Dataset: CIFAR10 I selected this dataset, because it contains a good level of complexity (10 classes, 3 color channels), at the same time it is also relatively small (low resolution (32x32 pixels),small datasize comparing to other deep learning datasets), this makes training models faster. I could train models very quick with different archtectures, optimizers, generalizations, learning rates, batch sizes.
Deep learning Model: 2D CNN 2D-CNN is designed specifically to process and analyze 2-dimensional data, such as images. This means it is particularly suitable for CIFAR10 image classification tasks due to its ability to effectively capture spatial patterns and hierarchies of features.
Deep learning Framework: PyTorch
# load libraries
import math
import numpy as np
import matplotlib.pyplot as plt
import torch
from torch import nn
import torch.optim as optim
from torch.utils.data import DataLoader
import torchvision
import torchvision.datasets as datasets
import torchvision.transforms as transforms
from torch.utils.data import Subset
from torchvision.transforms import ToTensor
import plotly.express as px
import wandb
api = wandb.Api()
wandb.login()
wandb: Currently logged in as: weiping-zhang (data_science2021). Use `wandb login --relogin` to force relogin
True
# Define transforms for data preprocessing
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4), # pad the image by 4 pixels padding=4, then randomly crop the image back to size 32x32
transforms.RandomHorizontalFlip(), # randomly flip image horizontally, to augment data
transforms.ToTensor(), # convert image to PyTorch tensor with shape (channels, height, width)
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) # normalize image values of each color channel to mean 0 and std 1
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
# Load the CIFAR-10 dataset
trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train)
testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)
Files already downloaded and verified Files already downloaded and verified
class_labels = trainset.classes
class_counts = torch.bincount(torch.tensor(trainset.targets))
class_counts_test = torch.bincount(torch.tensor(testset.targets))
plt.figure(figsize=(12, 3))
plt.bar(class_labels, class_counts)
plt.xlabel('Class Labels')
plt.ylabel('Number of Samples')
plt.title('Class Distribution (trainset)')
#plt.xticks(rotation=0)
plt.figure(figsize=(12, 3))
plt.bar(class_labels, class_counts_test)
plt.xlabel('Class Labels')
plt.ylabel('Number of Samples')
plt.title('Class Distribution (testset)')
plt.show()
# create a small train set with one batch and one small test set to quickly exam the model configurations
subset_indices = range(4)
trainset_small = Subset(trainset, subset_indices)
subset_indices = range(4)
testset_small = Subset(testset, subset_indices)
# small data loaders
trainloader_small = torch.utils.data.DataLoader(trainset_small, batch_size=4, shuffle=True, num_workers=2)
testloader_small = torch.utils.data.DataLoader(testset_small, batch_size=4, shuffle=False, num_workers=2)
# plot the pixel value distribution of the first images in the trainset_small set
for images, labels in trainloader_small:
image = images[0]
flattened_image = image.flatten()
# plot
plt.figure(figsize=(12, 3))
plt.hist(flattened_image.numpy(), bins = 256, range=(-1.5, 1.5))
plt.title('Pixel Value Distribution')
plt.xlabel('Pixel Value')
plt.ylabel('Frequency')
plt.show()
- improve convergence speed in training process and model stability.
- improve generalization
- prevent too large or too small gradients during backpropagation process.
evaluation metric: accurcy, as
it provides direct comparison between different models and approaches on the same dataset
train accuracy: is used to assess the model's ability to learn and fit trainset data
optimization criterion: cross entropy loss, which meansures the discrepancy between the predicted output and the true target values during training. It is often to optimize model by minimizing the loss.
# Define the 2D CNN architecture
class Net(nn.Module):
def __init__(self,config):
super(Net, self).__init__()
# 3 input channels, 32 output channels (32 filters), kernel size 3
self.conv1 = nn.Conv2d(3, 32, kernel_size=3)
self.conv2 = nn.Conv2d(32, 64, kernel_size=3)
self.relu = nn.ReLU() # keep only positive values, negative values are replaced with 0
self.pool = nn.MaxPool2d(2)
# flattened the output into a vector
self.fc1 = nn.Linear(64 * 14 * 14, 128)
self.fc2 = nn.Linear(128, 10)
self.dropout = nn.Dropout(p=config['dropout_rate'])
def forward(self, x):
# CONV1 - Relu - CONV2 - Relu - Maxpool - fc1 - Relu - Dropout - fc2 - softmax
x = self.relu(self.conv1(x))
x = self.pool(self.relu(self.conv2(x)))
# flatten the output into a vector
x = x.view(-1, 64 * 14 * 14)
x = self.relu(self.fc1(x))
x = self.dropout(x)
x = nn.functional.softmax(self.fc2(x), dim=1) # softmax: probability distribution over 10 classes
return x
def plot_results(train_losses, train_accuracies, test_accuracies, diff_accuracies):
epochs = range(1, len(train_losses) + 1)
plt.figure(figsize=(12, 4))
plt.subplot(1, 2, 1)
plt.plot(epochs, train_losses, label="Training Loss")
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.legend()
plt.subplot(1, 2, 2)
plt.plot(epochs, train_accuracies, label="Training Accuracy")
plt.plot(epochs, test_accuracies, label="Test Accuracy")
plt.plot(epochs, diff_accuracies, label="Accuracy Difference")
plt.xlabel("Epochs")
plt.ylabel("Accuracy")
plt.legend()
plt.show()
def cnn_train_evaluation(config):
wandb.init()
config = wandb.config
if config['data_size'] == 'small':
trainloader = trainloader_small
testloader = testloader_small
elif config['data_size'] == 'normal':
trainloader = torch.utils.data.DataLoader(trainset, batch_size = config.batch_size, shuffle=True, num_workers=2)
testloader = torch.utils.data.DataLoader(testset, batch_size = config.batch_size, shuffle=False, num_workers=2)
net = Net(config)
reg_type = config['regularization']
reg_strength = config['regularization_strength']
# Define the loss function, optimizer, regularization
criterion = nn.CrossEntropyLoss()
if config['optimizer'] == 'sgd':
if reg_type == 'l1':
optimizer = optim.SGD(net.parameters(), lr=config['learning_rate'], momentum=config['momentum'], weight_decay=reg_strength)
elif reg_type == 'l2':
optimizer = optim.SGD(net.parameters(), lr=config['learning_rate'], momentum=config['momentum'], weight_decay=reg_strength)
else:
optimizer = optim.SGD(net.parameters(), lr=config['learning_rate'], momentum=config['momentum'])
elif config['optimizer']== 'adam':
if reg_type == 'l1':
optimizer = optim.Adam(net.parameters(), lr=config['learning_rate'], weight_decay=reg_strength)
elif reg_type == 'l2':
optimizer = optim.Adam(net.parameters(), lr=config['learning_rate'], weight_decay=reg_strength)
else:
optimizer = optim.Adam(net.parameters(), lr=config['learning_rate'])
# append the loss, accuracy values in each epoch for plotting
train_losses = []
train_accuracies = []
test_accuracies = []
diff_accuracies = [] # Accuracy difference between train and testset
# add early stopping: if the loss not decrease for a certain epochs, then stop the training process
# initialize early stopping criterion
early_stop_epochs = config['early_stop_epochs']
early_stop_counter = 0
best_loss = float('inf')
# Train the model
for epoch in range(config['epochs']):
running_loss = 0.0
total_train = 0
correct_train = 0
for i, data in enumerate(trainloader, 0):
inputs, labels = data
optimizer.zero_grad()
outputs = net(inputs)
loss = criterion(outputs, labels)
if reg_type == 'l1':
l1_reg = torch.tensor(0.)
for param in net.parameters():
l1_reg += torch.norm(param, 1)
loss += reg_strength * l1_reg
elif reg_type == 'l2':
l2_reg = torch.tensor(0.)
for param in net.parameters():
l2_reg += torch.norm(param, 2)
loss += reg_strength * l2_reg
loss.backward()
optimizer.step()
running_loss += loss.item()
_, predicted_train = torch.max(outputs.data, 1)
total_train += labels.size(0)
correct_train += (predicted_train == labels).sum().item()
avg_loss = running_loss / len(trainloader)
# check for early stopping
if avg_loss < best_loss:
early_stop_counter = 0
best_loss = avg_loss
torch.save(net.state_dict(), 'best_model.pt')
else:
early_stop_counter += 1
if early_stop_counter >= early_stop_epochs:
print('Early stopping')
break
train_accuracy = correct_train / total_train
train_accuracies.append(train_accuracy)
train_losses.append(avg_loss)
wandb.log({"Training Loss": avg_loss}, step=epoch)
wandb.log({"Training Accuracy": train_accuracy}, step=epoch)
# Log test accuracy
correct = 0
total = 0
with torch.no_grad():
for data in testloader:
images, labels = data
outputs = net(images)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
test_accuracy = correct / total
wandb.log({"Test Accuracy": test_accuracy}, step=epoch)
test_accuracies.append(test_accuracy)
diff_accuracies.append(train_accuracy - test_accuracy)
wandb.log({"Accuracy difference": (train_accuracy - test_accuracy)}, step=epoch)
plot_results(train_losses, train_accuracies, test_accuracies, diff_accuracies)
config = {
"method": "grid",
"metric": {'name': 'Test Accuracy', 'goal': 'maximize'},
"parameters": {
"optimizer": {"values": ["sgd"]},
"data_size": {"values": ["small"]},
"learning_rate": {"values": [0.01]},
"momentum": {"values": [0]},
"epochs": {"values": [5000]},
"early_stop_epochs": {"values": [2000]}, # set a large number to leave out early stop here
"batch_size": {"values": [0]},
"dropout_rate": {"values": [0]},
"regularization": {"values": ["none"]},
"regularization_strength": {"values": [0]}
}}
# Initialize the sweep
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n-conv-layers_test", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: pqbkzumq Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-lr-bs_test/sweeps/pqbkzumq
wandb: Agent Starting Run: 2hasvv4m with config: wandb: batch_size: 0 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230427_195604-2hasvv4m
VBox(children=(Label(value='0.020 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▁▅▅▅▅▅▅▅▅▅▅████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▅▅▅▅▅▅▅▅▅▅████████████████████████████ |
| Training Loss | █▆▄▄▄▄▄▄▄▄▄▄▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.46127 |
./wandb/run-20230427_195604-2hasvv4m/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
I plan to try the following four architectures
For each architechture it is necessary to first tune learning rate and batch size, because each architechture may have different optimal learning rate and batch size
In this section, I will keep the other hyperparameters as following (not change):
optimizer: SGD (Stochastic Gradient Descent) is an commonly used optimization algorithm for training models in machine learning and deep learning trainings.
The updating step size = gradient * learning rate.
very small learning rate may lead to
very large learning rate may cause
In each epoch, SGD will randomly select a batch of the training data to compute the gradients and update the parameters to help preventing getting stuck in local optima.
config["parameters"]["data_size"]["values"] = ["normal"]
config["parameters"]["batch_size"]["values"] = [128,64,32,16]
config["parameters"]["learning_rate"]["values"] = [0.1,0.01,0.001]
config["parameters"]["epochs"]["values"] = [300]
config["parameters"]["early_stop_epochs"]["values"] = [20]
# Initialize the sweep
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-lr-bs", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: 66fs2rmd Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-lr-bs/sweeps/66fs2rmd
wandb: Agent Starting Run: bw4ki9ji with config: wandb: batch_size: 128 wandb: dropout: 0.01 wandb: epochs: 300 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/wandb/run-20230412_151223-bw4ki9ji
VBox(children=(Label(value='0.007 MB of 0.023 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.299410…
| Estimation error | ▁▁▂▂▁▃▃▃▃▃▄▄▅▄▆▄▅▆▅▅▅▆▅▅▆▆▇▇▇▇▆▇▇█▇▇▇▇█▇ |
| Test Accuracy | ▁▃▄▅▆▆▇▇▇▇▇▇▇▇▇█▇▇██████████████████████ |
| Training Accuracy | ▁▂▄▅▅▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇█▇█████████████████ |
| Training Loss | █▇▅▄▄▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Estimation error | 0.02536 |
| Test Accuracy | 0.7745 |
| Training Accuracy | 0.79986 |
| Training Loss | 1.66075 |
./wandb/run-20230412_151223-bw4ki9ji/logs
wandb: Agent Starting Run: mtjy7t6u with config: wandb: batch_size: 64 wandb: dropout: 0.01 wandb: epochs: 300 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/wandb/run-20230412_211417-mtjy7t6u
Early stopping
VBox(children=(Label(value='0.007 MB of 0.007 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Estimation error | ▁▃▃▄▅▄▅▃▄▄▄▄▅▅▅▆▄▅▄▄▄▄▅▅▅▄▅█▅▅▅▅▅▆▅▄▅▅▆▆ |
| Test Accuracy | ▁▃▄▅▅▆▆▇▇▇▇▇▇▇▇▇███████████▇█████▇██████ |
| Training Accuracy | ▁▃▄▅▆▆▆▇▇▇▇▇▇▇▇█████████████████████████ |
| Training Loss | █▆▅▄▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Estimation error | 0.00996 |
| Test Accuracy | 0.7094 |
| Training Accuracy | 0.71936 |
| Training Loss | 1.74057 |
./wandb/run-20230412_211417-mtjy7t6u/logs
wandb: Agent Starting Run: 1qkkafx9 with config: wandb: batch_size: 32 wandb: dropout: 0.01 wandb: epochs: 300 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/wandb/run-20230413_020013-1qkkafx9
Early stopping
VBox(children=(Label(value='0.007 MB of 0.007 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Estimation error | ▁▄▃▆▃▄▄▄▄▃▅▄▅▅▇▄▅▅▅▄▆▅▅▇▅▅▅▅▅▄▇▆▄▅▅▆█▃▃▆ |
| Test Accuracy | ▁▂▃▃▅▅▅▅▆▇▆▇▇▇▇▇███████▇██████▇▇▇▇▇▆▆▇▇▅ |
| Training Accuracy | ▁▃▄▄▄▅▅▆▆▇▇▇▇▇▇▇████████████████▇▇▇▇▇▇▇▆ |
| Training Loss | █▆▆▅▅▄▄▄▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂▂▂▂▂▂▂▃ |
| Estimation error | -0.02102 |
| Test Accuracy | 0.5306 |
| Training Accuracy | 0.50958 |
| Training Loss | 1.95105 |
./wandb/run-20230413_020013-1qkkafx9/logs
wandb: Agent Starting Run: 409sbjmv with config: wandb: batch_size: 16 wandb: dropout: 0.01 wandb: epochs: 300 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/wandb/run-20230413_053844-409sbjmv
Early stopping
VBox(children=(Label(value='0.007 MB of 0.007 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Estimation error | ▁▄▅█▃▄▄▅▅▅▅▅▅▅▅▅▅▅▅▅▅ |
| Test Accuracy | █▇▆▂▃▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▆█▇▅▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Loss | ▂▁▂▄▇▇▇██████████████ |
| Estimation error | 0.0 |
| Test Accuracy | 0.1 |
| Training Accuracy | 0.1 |
| Training Loss | 2.36115 |
./wandb/run-20230413_053844-409sbjmv/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
%wandb weiping-zhang/mc1-sgd-early-stop-lr-bs/reports/3-1-1-1-tune-learning-rate-batch-size--Vmlldzo0NDc0Njg5 -h 1024
# use 500 epochs of 2 convolutional layers
# update learning_rate to 0.01, batch_size to 64
config["parameters"]["learning_rate"]["values"] = [0.01]
config["parameters"]["batch_size"]["values"] = [64]
config["parameters"]["epochs"]["values"] = [500]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n_conv_layers", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
%wandb weiping-zhang/mc1-sgd-early-stop-n_conv_layers/reports/2-convolutional-layers--Vmlldzo0NDg0ODUy -h 1024
# use 0 convolutional layers, 3 fully connected layers
class Net(nn.Module):
def __init__(self,config):
super(Net, self).__init__()
self.fc1 = nn.Linear(3 * 32 * 32, 256)
self.fc2 = nn.Linear(256, 128)
self.fc3 = nn.Linear(128, 10)
self.relu = nn.ReLU()
self.dropout = nn.Dropout(p=config['dropout_rate'])
def forward(self, x):
# fc1 - Relu - Dropout - fc2 - Relu - Dropout - fc3 - softmax
x = torch.flatten(x, start_dim=1)
x = self.relu(self.fc1(x))
x = self.dropout(x)
x = self.relu(self.fc2(x))
x = self.dropout(x)
x = nn.functional.softmax(self.fc3(x), dim=1)
return x
config["parameters"]["data_size"]["values"] = ['small']
config["parameters"]["epochs"]["values"] = [2000]
config["parameters"]["early_stop_epochs"]["values"] = [500]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n-conv-layers_test", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
wandb: WARNING Calling wandb.login() after wandb.init() has no effect.
Create sweep with ID: qozzlbto Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-n-conv-layers_test/sweeps/qozzlbto
wandb: Waiting for W&B process to finish... (success).
/home/jovyan/work/persistent/wandb/run-20230427_212640-do0ckhtf
wandb: 🚀 View run trim-sweep-1 at: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-lr-bs/runs/do0ckhtf wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) wandb: Find logs at: ./wandb/run-20230427_212640-do0ckhtf/logs wandb: Agent Starting Run: r3xgc6x8 with config: wandb: batch_size: 128 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0
/home/jovyan/work/persistent/wandb/run-20230427_212657-r3xgc6x8
VBox(children=(Label(value='0.005 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.222829…
| Accuracy difference | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▁▅▅▅▅▅▅▅▅▅▅▅▅████▅██████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▅▅▁▅▅▅▅▅▅▅▅▅▅▅▅████▅██████ |
| Training Loss | ████████▇▇▇▇▆▆▅▄▅▄▄▃▃▃▃▃▃▃▃▃▃▂▂▂▂▂▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.5182 |
./wandb/run-20230427_212657-r3xgc6x8/logs
wandb: Agent Starting Run: ygr81q7e with config: wandb: batch_size: 128 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0
/home/jovyan/work/persistent/wandb/run-20230427_215248-ygr81q7e
| Accuracy difference | ▁▁▁█████████████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▁█████████████████████████████████████ |
| Training Loss | █▆▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.46168 |
./wandb/run-20230427_215248-ygr81q7e/logs
wandb: Agent Starting Run: yr2uo80q with config: wandb: batch_size: 128 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0
/home/jovyan/work/persistent/wandb/run-20230427_221419-yr2uo80q
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185482…
| Accuracy difference | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Loss | ▄█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.46115 |
./wandb/run-20230427_221419-yr2uo80q/logs
wandb: Agent Starting Run: ymp7pna5 with config: wandb: batch_size: 64 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0
/home/jovyan/work/persistent/wandb/run-20230427_223615-ymp7pna5
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185427…
| Accuracy difference | ▁▄▅▅▅█▅▇▇▅▇▇▅▅▅▇▇▇▇▇▇▇▇▇▇▇██████████████ |
| Test Accuracy | █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▃▅▅▅█▅▆▆▅▆▆▅▅▅▆▆▆▆▆▆▆▆▆▆▆██████████████ |
| Training Loss | █████████▇▇▆▆▆▅▅▄▄▃▃▃▃▃▂▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.49739 |
./wandb/run-20230427_223615-ymp7pna5/logs
wandb: Agent Starting Run: uw0zzpji with config: wandb: batch_size: 64 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0
/home/jovyan/work/persistent/wandb/run-20230427_225829-uw0zzpji
VBox(children=(Label(value='0.005 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.222720…
| Accuracy difference | ▁▁▁█████████████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▁█████████████████████████████████████ |
| Training Loss | ██▅▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.46132 |
./wandb/run-20230427_225829-uw0zzpji/logs
wandb: Agent Starting Run: hwmkx1av with config: wandb: batch_size: 64 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0
/home/jovyan/work/persistent/wandb/run-20230427_232116-hwmkx1av
VBox(children=(Label(value='0.004 MB of 0.009 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.431442…
| Accuracy difference | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Loss | █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.46117 |
./wandb/run-20230427_232116-hwmkx1av/logs
wandb: Agent Starting Run: 1j1ko1uj with config: wandb: batch_size: 32 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0
/home/jovyan/work/persistent/wandb/run-20230427_234406-1j1ko1uj
VBox(children=(Label(value='0.020 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▁▁▂▄▅▅▅▅▅▇▅▅▅▅▅▅▅▅▅▅▇▇▇▇▇▇▇▇▇▇▇▇▇██████ |
| Test Accuracy | █████▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▁▃▅▅▅▅▅▅▆▅▅▅▅▅▅▅▅▅▅▆▆▆▆▆▆▆▆▆▆▆▆▆██████ |
| Training Loss | █████████▇▇▇▆▆▅▅▄▄▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▁▂▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.55514 |
./wandb/run-20230427_234406-1j1ko1uj/logs
wandb: Agent Starting Run: w174y19v with config: wandb: batch_size: 32 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0
/home/jovyan/work/persistent/wandb/run-20230428_000711-w174y19v
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185508…
| Accuracy difference | ▁▁▅█████████████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▅█████████████████████████████████████ |
| Training Loss | █▇▄▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.46129 |
./wandb/run-20230428_000711-w174y19v/logs
wandb: Agent Starting Run: qhoz5cv6 with config: wandb: batch_size: 32 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0
/home/jovyan/work/persistent/wandb/run-20230428_003019-qhoz5cv6
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185491…
| Accuracy difference | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Loss | █▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.46116 |
./wandb/run-20230428_003019-qhoz5cv6/logs
wandb: Agent Starting Run: 3431g2wn with config: wandb: batch_size: 16 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0
/home/jovyan/work/persistent/wandb/run-20230428_005347-3431g2wn
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185444…
| Accuracy difference | ▁▁▃▃▆▆█▆▆▆▆▆▆▆▆▃▆▆▆▆▆▆▆▆▆███▆▆██████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▃▃▆▆█▆▆▆▆▆▆▆▆▃▆▆▆▆▆▆▆▆▆███▆▆██████████ |
| Training Loss | ██████████▇▇▇▇▅▅▄▃▄▃▃▃▃▃▃▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.51169 |
./wandb/run-20230428_005347-3431g2wn/logs
wandb: Agent Starting Run: ce5vjgc3 with config: wandb: batch_size: 16 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0
/home/jovyan/work/persistent/wandb/run-20230428_011743-ce5vjgc3
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185444…
| Accuracy difference | █▁▁█████████████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | █▁▁█████████████████████████████████████ |
| Training Loss | █▇▄▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.46136 |
./wandb/run-20230428_011743-ce5vjgc3/logs
wandb: Agent Starting Run: d0mlgm14 with config: wandb: batch_size: 16 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0
/home/jovyan/work/persistent/wandb/run-20230428_014125-d0mlgm14
VBox(children=(Label(value='0.004 MB of 0.009 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.421236…
| Accuracy difference | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Loss | █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.46116 |
./wandb/run-20230428_014125-d0mlgm14/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
Error in callback <function _WandbInit._pause_backend at 0x7f478dce9bd0> (for post_run_cell):
--------------------------------------------------------------------------- BrokenPipeError Traceback (most recent call last) File /opt/conda/lib/python3.10/site-packages/backcall/backcall.py:104, in callback_prototype.<locals>.adapt.<locals>.adapted(*args, **kwargs) 102 kwargs.pop(name) 103 # print(args, kwargs, unmatched_pos, cut_positional, unmatched_kw) --> 104 return callback(*args, **kwargs) File /opt/conda/lib/python3.10/site-packages/wandb/sdk/wandb_init.py:419, in _WandbInit._pause_backend(self) 417 if self.backend.interface is not None: 418 logger.info("pausing backend") # type: ignore --> 419 self.backend.interface.publish_pause() File /opt/conda/lib/python3.10/site-packages/wandb/sdk/interface/interface.py:665, in InterfaceBase.publish_pause(self) 663 def publish_pause(self) -> None: 664 pause = pb.PauseRequest() --> 665 self._publish_pause(pause) File /opt/conda/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py:340, in InterfaceShared._publish_pause(self, pause) 338 def _publish_pause(self, pause: pb.PauseRequest) -> None: 339 rec = self._make_request(pause=pause) --> 340 self._publish(rec) File /opt/conda/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py:51, in InterfaceSock._publish(self, record, local) 49 def _publish(self, record: "pb.Record", local: Optional[bool] = None) -> None: 50 self._assign(record) ---> 51 self._sock_client.send_record_publish(record) File /opt/conda/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py:221, in SockClient.send_record_publish(self, record) 219 server_req = spb.ServerRequest() 220 server_req.record_publish.CopyFrom(record) --> 221 self.send_server_request(server_req) File /opt/conda/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py:155, in SockClient.send_server_request(self, msg) 154 def send_server_request(self, msg: Any) -> None: --> 155 self._send_message(msg) File /opt/conda/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py:152, in SockClient._send_message(self, msg) 150 header = struct.pack("<BI", ord("W"), raw_size) 151 with self._lock: --> 152 self._sendall_with_error_handle(header + data) File /opt/conda/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py:130, in SockClient._sendall_with_error_handle(self, data) 128 start_time = time.monotonic() 129 try: --> 130 sent = self._sock.send(data) 131 # sent equal to 0 indicates a closed socket 132 if sent == 0: BrokenPipeError: [Errno 32] Broken pipe
config["parameters"]["data_size"]["values"] = ['normal']
config["parameters"]["epochs"]["values"] = [50]
config["parameters"]["learning_rate"]["values"] = [0.001,0.01,0.1]
config["parameters"]["batch_size"]["values"] = [128,64,32,16]
config["parameters"]["early_stop_epochs"]["values"] = [20]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-lr-bs", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
entity, project, sweep_id = "weiping-zhang","mc1-sgd-early-stop-lr-bs", "jqd8f38v"
sweep = api.sweep(f"{entity}/{project}/{sweep_id}")
sweep.display(height=1024)
True
config["parameters"]["data_size"]["values"] = ['normal']
config["parameters"]["learning_rate"]["values"] = [0.01]
config["parameters"]["batch_size"]["values"] = [32]
config["parameters"]["epochs"]["values"] = [500]
config["parameters"]["early_stop_epochs"]["values"] = [20]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n_conv_layers", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: wzt3e13v Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-n_conv_layers/sweeps/wzt3e13v
wandb: Agent Starting Run: 7qz1k8du with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 500 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230428_105103-7qz1k8du
wandb: Network error (ConnectionError), entering retry loop.
sweep = api.sweep(f"weiping-zhang/mc1-sgd-early-stop-n_conv_layers/sweeps/wzt3e13v")
sweep.display(height=1024)
True
# use 1 convolutional layers
class Net(nn.Module):
def __init__(self,config):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, 32, kernel_size=3)
self.relu = nn.ReLU()
self.pool = nn.MaxPool2d(2)
self.fc1 = nn.Linear(32 * 15 * 15, 128)
self.fc2 = nn.Linear(128, 10)
self.dropout = nn.Dropout(p=config['dropout_rate'])
def forward(self, x):
# CONV1 - Relu - Maxpool - fc1 - Relu - Dropout - fc2 - softmax
x = self.pool(self.relu(self.conv1(x)))
x = torch.flatten(x, start_dim=1)
x = self.relu(self.fc1(x))
x = self.dropout(x)
x = nn.functional.softmax(self.fc2(x), dim=1)
return x
config["parameters"]["data_size"]["values"] = ['small']
config["parameters"]["epochs"]["values"] = [2000]
config["parameters"]["early_stop_epochs"]["values"] = [500]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n-conv-layers_test", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: 9plpr0j2 Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-n-conv-layers_test/sweeps/9plpr0j2
wandb: Agent Starting Run: lb2idy4e with config: wandb: batch_size: 0 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 3000 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning. wandb: Currently logged in as: weiping-zhang. Use `wandb login --relogin` to force relogin
/home/jovyan/work/persistent/wandb/run-20230428_181734-lb2idy4e
VBox(children=(Label(value='0.020 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▅██████████████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▅██████████████████████████████████████ |
| Training Loss | █▄▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.46143 |
./wandb/run-20230428_181734-lb2idy4e/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
# tune learning_rate & batch_size
config["parameters"]["data_size"]["values"] = ['normal']
config["parameters"]["epochs"]["values"] = [50]
config["parameters"]["learning_rate"]["values"] = [0.001,0.01,0.1]
config["parameters"]["batch_size"]["values"] = [128,64,32,16]
config["parameters"]["early_stop_epochs"]["values"] = [20]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-lr-bs", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: soly2oso Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-lr-bs/sweeps/soly2oso
wandb: Agent Starting Run: va08qpgf with config: wandb: batch_size: 128 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 50 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230428_183306-va08qpgf
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185756…
| Accuracy difference | █▆▄▂▂▂▁▂▁▁▁▁▁▁▁▁▂▃▃▃▄▅▅▅▅▅▅▅▅▅▄▄▄▄▄▄▃▃▂▃ |
| Test Accuracy | ▁▂▄▅▆▆▆▆▆▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇████ |
| Training Accuracy | ▁▂▃▄▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇███ |
| Training Loss | ████████▇▇▇▇▇▇▇▆▆▆▆▆▅▅▅▅▄▄▄▄▃▃▃▃▂▂▂▂▂▁▁▁ |
| Accuracy difference | -0.01616 |
| Test Accuracy | 0.2814 |
| Training Accuracy | 0.26524 |
| Training Loss | 2.20209 |
./wandb/run-20230428_183306-va08qpgf/logs
wandb: Agent Starting Run: 5cjfro95 with config: wandb: batch_size: 128 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 50 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.016669166183661824, max=1.0…
/home/jovyan/work/persistent/wandb/run-20230428_185536-5cjfro95
VBox(children=(Label(value='0.004 MB of 0.009 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.421035…
| Accuracy difference | ▁██▇▅▆▇▆▇▇█▅▇▆▆▇▆▆▆▇▆▇▆▇▇▇▆▆▆▆▇▅▆▆▇▆▇▆▆▆ |
| Test Accuracy | ▁▁▁▂▃▃▃▄▄▄▄▅▅▅▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇█████ |
| Training Accuracy | ▁▂▂▂▃▃▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇█████ |
| Training Loss | ██▇▆▆▅▅▅▅▄▄▄▄▄▄▃▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁ |
| Accuracy difference | -0.02736 |
| Test Accuracy | 0.4825 |
| Training Accuracy | 0.45514 |
| Training Loss | 2.01132 |
./wandb/run-20230428_185536-5cjfro95/logs
wandb: Sweep Agent: Waiting for job. wandb: Job received. wandb: Agent Starting Run: jj641jd2 with config: wandb: batch_size: 128 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 50 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230428_191801-jj641jd2
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185791…
| Accuracy difference | ▁▃▅▅▆▅▅▄▆▄▆▄▅▅▅▅▆▆▅▅▆▅▅▅▇▆▆▆▆▅▅▆▆▅▅▆█▇▆▇ |
| Test Accuracy | ▁▂▃▃▄▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇████████████ |
| Training Accuracy | ▁▃▃▄▄▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇████████████ |
| Training Loss | █▇▆▅▅▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.0175 |
| Test Accuracy | 0.6522 |
| Training Accuracy | 0.6347 |
| Training Loss | 1.82625 |
./wandb/run-20230428_191801-jj641jd2/logs
wandb: Agent Starting Run: igk84odl with config: wandb: batch_size: 64 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 50 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230428_193932-igk84odl
VBox(children=(Label(value='0.008 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.392640…
| Accuracy difference | ▅▅▁▁▂▃▅▆█▇▇▇▇▇▇████▆▅▅▅▅▃▃▂▄▃▅▃▄▅▄▄▅▅▄▄▅ |
| Test Accuracy | ▁▂▃▄▄▄▄▄▄▄▄▄▄▄▅▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇█████████ |
| Training Accuracy | ▁▂▂▃▄▄▄▄▄▄▄▄▄▅▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇████████ |
| Training Loss | ██████▇▇▇▇▆▆▅▅▅▅▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁ |
| Accuracy difference | -0.01512 |
| Test Accuracy | 0.3167 |
| Training Accuracy | 0.30158 |
| Training Loss | 2.15938 |
./wandb/run-20230428_193932-igk84odl/logs
wandb: Agent Starting Run: ics4z9iv with config: wandb: batch_size: 64 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 50 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230428_200642-ics4z9iv
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185756…
| Accuracy difference | ▁▂▄▂▄▄▅▆▄▅▇▇▇▅▄▄▅▃▅▇▇▆▄▃▃▅▄▇▅▅▅▆█▆▅▆▆▆▅▇ |
| Test Accuracy | ▁▂▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇███▇███████ |
| Training Accuracy | ▁▂▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇█████████ |
| Training Loss | █▇▆▆▅▅▅▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.02042 |
| Test Accuracy | 0.5618 |
| Training Accuracy | 0.54138 |
| Training Loss | 1.92493 |
./wandb/run-20230428_200642-ics4z9iv/logs
wandb: Agent Starting Run: kd6os70a with config: wandb: batch_size: 64 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 50 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230428_203513-kd6os70a
| Accuracy difference | ▁▄▄▂▃▂▄▃▃▄▄▅█▄▃▅▃▃▄▃▃▃▃▃▄▃▄▃▃▅▆▄▄▄▄▅▃▃▄▃ |
| Test Accuracy | ▁▂▃▄▅▅▅▅▆▆▆▆▄▆▇▆▇▇▇▇█▇██▇█▇██▇▇████▇████ |
| Training Accuracy | ▁▃▄▄▅▅▅▆▆▆▆▇▇▇▇▇▇▇▇▇▇█▇█████████████████ |
| Training Loss | █▆▅▅▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.02522 |
| Test Accuracy | 0.6629 |
| Training Accuracy | 0.63768 |
| Training Loss | 1.82136 |
./wandb/run-20230428_203513-kd6os70a/logs
wandb: Agent Starting Run: pvb5xxmc with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 50 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230428_210302-pvb5xxmc
VBox(children=(Label(value='0.005 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.223036…
| Accuracy difference | ▆▃▁▄█▇▄▇▆▆▄▄▄▅▅▄▅▃▃▃▄▅▅▅▅▅▅▆▆▄▅▆▄▄▆▆▄▄▅▆ |
| Test Accuracy | ▁▂▂▃▃▃▄▄▄▅▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇████████ |
| Training Accuracy | ▁▁▂▂▃▃▄▄▄▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇██████ |
| Training Loss | ████▇▇▆▆▅▅▅▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁ |
| Accuracy difference | -0.01632 |
| Test Accuracy | 0.3778 |
| Training Accuracy | 0.36148 |
| Training Loss | 2.10262 |
./wandb/run-20230428_210302-pvb5xxmc/logs
wandb: Agent Starting Run: 96uznue9 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 50 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230428_213857-96uznue9
VBox(children=(Label(value='0.004 MB of 0.005 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.833612…
| Accuracy difference | ▁▅▇▇▆▆▇▇▆▆▆█▆▆▇▇█▆▇▇▇▇▇▇▇▇▇▇▇██▇▆▇▆▇▇█▇█ |
| Test Accuracy | ▁▂▃▃▄▄▄▄▅▅▆▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇█████████ |
| Training Accuracy | ▁▃▃▄▄▄▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Loss | █▇▆▆▅▅▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.02178 |
| Test Accuracy | 0.6183 |
| Training Accuracy | 0.59652 |
| Training Loss | 1.8681 |
./wandb/run-20230428_213857-96uznue9/logs
wandb: Agent Starting Run: z0zy6gc0 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 50 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230428_221554-z0zy6gc0
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▅▇▇▅▆▆▅▇▆▆▆▇▆▅▇█▆▇▆▇█▇▇▇▆█▆▅█▆▇▇▇▅▅▆▆ |
| Test Accuracy | ▁▃▃▄▆▆▆▇▆▇▇█▇██▇▇█▇██▇▇▇▇█▇██▇█▇▇▇██▇█ |
| Training Accuracy | ▁▄▅▆▆▆▇▇▇▇████████████████████████████ |
| Training Loss | █▅▄▃▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.04678 |
| Test Accuracy | 0.575 |
| Training Accuracy | 0.52822 |
| Training Loss | 1.93162 |
./wandb/run-20230428_221554-z0zy6gc0/logs
wandb: Agent Starting Run: bl1js2ru with config: wandb: batch_size: 16 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 50 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230428_224559-bl1js2ru
| Accuracy difference | ▁▆▆█▇▆▆▅▅▄▄▅▅▅▆▆▆▆▆▆▆▆▇▆▆▆▇▆█▆█▇▇▇▇▇▇▇▆▆ |
| Test Accuracy | ▁▁▁▂▃▃▄▄▅▅▅▅▅▅▆▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇████ |
| Training Accuracy | ▁▂▂▃▃▄▄▄▅▅▅▅▆▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇████████ |
| Training Loss | ██▇▇▆▅▅▅▄▄▄▄▃▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.02258 |
| Test Accuracy | 0.4506 |
| Training Accuracy | 0.42802 |
| Training Loss | 2.03677 |
./wandb/run-20230428_224559-bl1js2ru/logs
wandb: Sweep Agent: Waiting for job. wandb: Job received. wandb: Agent Starting Run: wgqel0sv with config: wandb: batch_size: 16 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 50 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230428_233945-wgqel0sv
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185774…
| Accuracy difference | ▁▄▅▅▇▄▅▅▆▄▅▅▅▅▆▆▆▇▆▆▆█▆▆▆▆▆▇▆▇▆▇█▇▆▇▇▇▇▇ |
| Test Accuracy | ▁▂▃▃▄▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇█▇████████████ |
| Training Accuracy | ▁▂▃▄▄▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇████████████ |
| Training Loss | █▇▆▅▅▄▄▄▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.02126 |
| Test Accuracy | 0.6332 |
| Training Accuracy | 0.61194 |
| Training Loss | 1.85013 |
./wandb/run-20230428_233945-wgqel0sv/logs
wandb: Agent Starting Run: a7eol2j6 with config: wandb: batch_size: 16 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 50 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_003049-a7eol2j6
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▃▅▂▄▄█▂▇▆▂▅▄▄▄▆▁▁▆▁▅▅▄▅ |
| Test Accuracy | ▄▆▆█▇▇▅▇▅▅▆▅▅▅▄▃▄▄▁▃▁▂▂▂ |
| Training Accuracy | ▂▆▇████▇▇▇▆▆▅▄▄▄▂▃▂▁▁▂▂▃ |
| Training Loss | ▆▃▂▁▁▁▁▂▂▂▃▃▄▅▅▅▇▆▇██▇▇▆ |
| Accuracy difference | -0.02444 |
| Test Accuracy | 0.3556 |
| Training Accuracy | 0.33116 |
| Training Loss | 2.12991 |
./wandb/run-20230429_003049-a7eol2j6/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
project, sweep_id = "mc1-sgd-early-stop-lr-bs", "soly2oso"
sweep = api.sweep(f"{entity}/{project}/{sweep_id}")
sweep.display(height=500)
True
# update learning rate & batch size, train on normal dataset
config["parameters"]["learning_rate"]["values"] = [0.01]
config["parameters"]["batch_size"]["values"] = [32]
config["parameters"]["epochs"]["values"] = [500]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n_conv_layers", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: ssah2iar Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-n_conv_layers/sweeps/ssah2iar
wandb: Agent Starting Run: 7ih1lrmb with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 500 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_005752-7ih1lrmb
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185748…
| Accuracy difference | ▂▁▁▃▁▂▂▃▃▃▄▄▄▄▄▆▄▆▅▅▅▆▆▆▇▆▇▆▆▆▆▇▆█▇▇▇█▇▇ |
| Test Accuracy | ▁▄▅▅▆▆▇▇▇▇▇▇▇▇▇▇█▇██████████████████████ |
| Training Accuracy | ▁▃▄▅▅▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇█████████████████ |
| Training Loss | █▆▅▄▄▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.02376 |
| Test Accuracy | 0.7448 |
| Training Accuracy | 0.76856 |
| Training Loss | 1.69334 |
./wandb/run-20230429_005752-7ih1lrmb/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
sweep = api.sweep(f"weiping-zhang/mc1-sgd-early-stop-n_conv_layers/sweeps/ssah2iar")
sweep.display(height=1024)
True
# use 3 convolutional layers
class Net(nn.Module):
def __init__(self,config):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, 32, kernel_size=3)
self.conv2 = nn.Conv2d(32, 64, kernel_size=3)
self.conv3 = nn.Conv2d(64,256,kernel_size=3)
self.relu = nn.ReLU()
self.pool = nn.MaxPool2d(2)
self.fc1 = nn.Linear(256 * 6 * 6, 128)
self.fc2 = nn.Linear(128, 10)
self.dropout = nn.Dropout(p=config['dropout_rate'])
def forward(self, x):
# CONV1 - Relu - CONV2 - Relu - Maxpool - CONV3 - Relu - Maxpool - fc1 - Relu - Dropout - fc2 - softmax
x = self.relu(self.conv1(x))
x = self.pool(self.relu(self.conv2(x)))
x = self.pool(self.relu(self.conv3(x)))
x = torch.flatten(x, start_dim=1)
x = self.relu(self.fc1(x))
x = self.dropout(x)
x = nn.functional.softmax(self.fc2(x), dim=1)
return x
# test on small dataset to check if the model is able to learn
config["parameters"]["data_size"]["values"] = ['small']
config["parameters"]["epochs"]["values"] = [2000]
config["parameters"]["early_stop_epochs"]["values"] = [500]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n-conv-layers_test", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: 98cqz57j Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-n-conv-layers_test/sweeps/98cqz57j
wandb: Agent Starting Run: 614jgmpg with config: wandb: batch_size: 32 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_064547-614jgmpg
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185930…
| Accuracy difference | ▅▁▁▅████████████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▅▁▁▅████████████████████████████████████ |
| Training Loss | █▇▅▄▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.46144 |
./wandb/run-20230429_064547-614jgmpg/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
# tune learning_rate & batch_size
config["parameters"]["data_size"]["values"] = ['normal']
config["parameters"]["epochs"]["values"] = [100]
config["parameters"]["learning_rate"]["values"] = [0.001,0.01,0.1]
config["parameters"]["batch_size"]["values"] = [128,64,32,16]
config["parameters"]["early_stop_epochs"]["values"] = [20]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-lr-bs", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: 7tmfsvmo Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-lr-bs/sweeps/7tmfsvmo
wandb: Agent Starting Run: sio95ncq with config: wandb: batch_size: 128 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 100 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_081942-sio95ncq
VBox(children=(Label(value='0.008 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.392585…
| Accuracy difference | ▆█▇▅▅▅▆▆▅▅▅▆▆▆▅▃▃▁▂▃▃▄▄▆▆▅▄▄▃▄▅▄▄▄▅▄▃▃▁▃ |
| Test Accuracy | ▁▁▂▃▄▄▄▄▄▄▄▄▆▇███▇▇▆▅▄▄▃▃▂▂▃▃▃▄▄▅▅▅▅▅▅▆▇ |
| Training Accuracy | ▁▁▂▃▄▄▄▄▄▄▄▄▆▇███▇▆▅▅▄▃▃▃▂▂▂▃▃▄▄▄▅▅▅▄▄▅▆ |
| Training Loss | ██████████████████████▇▇▇▇▇▇▇▇▇▆▆▆▅▄▄▃▂▁ |
| Accuracy difference | -0.00668 |
| Test Accuracy | 0.1598 |
| Training Accuracy | 0.15312 |
| Training Loss | 2.27367 |
./wandb/run-20230429_081942-sio95ncq/logs
wandb: Agent Starting Run: g2x17ssn with config: wandb: batch_size: 128 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 100 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_092502-g2x17ssn
VBox(children=(Label(value='0.008 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.392695…
| Accuracy difference | ▆▆▄▇█▄▄▃▂▃▃▄▃▃▃▄▁▂▃▄▅▂▄▁▂▂▃▂▄▄▃▄▅▄▃▃▅▅▅▃ |
| Test Accuracy | ▁▂▂▂▂▃▃▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇██████ |
| Training Accuracy | ▁▁▂▂▂▃▃▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇██████ |
| Training Loss | █████▇▆▆▅▅▅▅▄▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁ |
| Accuracy difference | -0.0316 |
| Test Accuracy | 0.59 |
| Training Accuracy | 0.5584 |
| Training Loss | 1.90368 |
./wandb/run-20230429_092502-g2x17ssn/logs
wandb: Agent Starting Run: 0jm10odx with config: wandb: batch_size: 128 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 100 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_103038-0jm10odx
| Accuracy difference | ▁▄▅▃▆▅▃▄▅▄▅▄▄▇▅▅▅▇▇▆▇▅▆▇▆▆█▆▇▇▇▇▆▇▇▆▇█▇▇ |
| Test Accuracy | ▁▂▃▄▄▅▅▆▆▆▆▇▇▆▇▇▇▇▇▇▇█▇▇██▇█████████████ |
| Training Accuracy | ▁▃▄▄▄▅▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇███████████████ |
| Training Loss | █▆▆▅▅▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.02314 |
| Test Accuracy | 0.8018 |
| Training Accuracy | 0.82494 |
| Training Loss | 1.63614 |
./wandb/run-20230429_103038-0jm10odx/logs
wandb: Sweep Agent: Waiting for job. wandb: Job received. wandb: Agent Starting Run: ivczmyrh with config: wandb: batch_size: 64 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 100 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_113725-ivczmyrh
VBox(children=(Label(value='0.004 MB of 0.009 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.421035…
| Accuracy difference | █▆▇███▇▆▅▅▆▇▇▆▅▄▄▄▄▄▅▄▃▄▃▃▃▃▃▃▂▃▂▂▁▁▂▂▂▁ |
| Test Accuracy | ▁▂▁▁▁▁▁▂▂▃▃▃▃▃▃▄▄▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇██████ |
| Training Accuracy | ▁▂▁▁▁▁▁▂▂▃▃▃▃▃▃▃▄▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇██████ |
| Training Loss | █████████████▇▇▇▆▆▅▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁ |
| Accuracy difference | -0.03644 |
| Test Accuracy | 0.3657 |
| Training Accuracy | 0.32926 |
| Training Loss | 2.12616 |
./wandb/run-20230429_113725-ivczmyrh/logs
wandb: Agent Starting Run: 10bevwe9 with config: wandb: batch_size: 64 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 100 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_130427-10bevwe9
| Accuracy difference | ▄▇▄▄▃▄▃▄▃▂▁▂▆▃▅▅▃▃▄▃▃▅▆▅▄▅▃▅▃▃▄█▄▇▅▅▅▅▇▄ |
| Test Accuracy | ▁▁▃▃▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇█▇██████ |
| Training Accuracy | ▁▁▃▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇█████████ |
| Training Loss | ██▇▆▆▆▅▅▅▅▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.02638 |
| Test Accuracy | 0.6933 |
| Training Accuracy | 0.66692 |
| Training Loss | 1.79498 |
./wandb/run-20230429_130427-10bevwe9/logs
wandb: Sweep Agent: Waiting for job. wandb: Job received. wandb: Agent Starting Run: m0j232q9 with config: wandb: batch_size: 64 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 100 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_143042-m0j232q9
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▂▃▃▄▅▃▄▃▇▅▄▄▄▄█▆▅▄▅▅▅▅▅▅▄▅▅▅▄▄▅▇▅▄▅▅▅▄▅ |
| Test Accuracy | ▁▃▄▄▅▅▆▆▇▅▆▇▇▇▇▆▇▇█▇▇█▇█████████▇███████ |
| Training Accuracy | ▁▃▄▄▅▅▆▆▆▇▇▇▇▇▇▇▇███████████████████████ |
| Training Loss | █▆▅▅▄▄▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.00526 |
| Test Accuracy | 0.7878 |
| Training Accuracy | 0.78254 |
| Training Loss | 1.67788 |
./wandb/run-20230429_143042-m0j232q9/logs
wandb: Sweep Agent: Waiting for job. wandb: Job received. wandb: Agent Starting Run: ctnpm7y6 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 100 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_160508-ctnpm7y6
VBox(children=(Label(value='0.016 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.781736…
| Accuracy difference | ▇▆▆▅▅▅▅▅▅▅█▇▄▄▄▃▃▂▂▂▂▂▁▁▁▂▂▁▂▁▂▂▂▂▂▂▂▂▂▂ |
| Test Accuracy | ▁▂▃▃▃▃▃▃▃▄▃▂▄▄▅▅▅▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇████████ |
| Training Accuracy | ▁▂▃▃▃▃▃▃▃▄▃▃▃▄▅▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇███████ |
| Training Loss | ███████████▇▆▅▅▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁ |
| Accuracy difference | -0.03142 |
| Test Accuracy | 0.4189 |
| Training Accuracy | 0.38748 |
| Training Loss | 2.06981 |
./wandb/run-20230429_160508-ctnpm7y6/logs
wandb: Agent Starting Run: 5ucpygf4 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 100 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_175325-5ucpygf4
VBox(children=(Label(value='0.004 MB of 0.009 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.420974…
| Accuracy difference | ▅▂▃▂▃▃▁▂▁▄▃▃▂▂▁▅▂▄▅▆▅▃▃▃▅▅▅▆▄▄▄▆▅▅▇▅█▇█▇ |
| Test Accuracy | ▁▂▃▃▄▄▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇████████████ |
| Training Accuracy | ▁▁▃▃▄▄▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Loss | ██▇▆▅▅▅▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.00224 |
| Test Accuracy | 0.7632 |
| Training Accuracy | 0.76096 |
| Training Loss | 1.70115 |
./wandb/run-20230429_175325-5ucpygf4/logs
wandb: Agent Starting Run: t70i2zz1 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 100 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_194431-t70i2zz1
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▄▁▃▄▂▃▃▄▃▇▅▂▄▄▃▄▄▄▃▄▄▅▁▅▅▅▄▆▅█▆▆▆▆▆ |
| Test Accuracy | ▃▅▅▆▆▆▇▇▇▇▇█████████▇▇█▆▅▅▃▂▃▁▁▁▁▁▁ |
| Training Accuracy | ▃▄▅▅▆▆▇▇▇▇████████████▇▇▆▅▂▂▃▂▁▁▁▁▁ |
| Training Loss | ▆▅▄▃▃▃▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▂▂▃▄▇▇▆▇█████ |
| Accuracy difference | -0.0002 |
| Test Accuracy | 0.1015 |
| Training Accuracy | 0.1013 |
| Training Loss | 2.35984 |
./wandb/run-20230429_194431-t70i2zz1/logs
wandb: Agent Starting Run: 83b57mw5 with config: wandb: batch_size: 16 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 100 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_202907-83b57mw5
VBox(children=(Label(value='0.004 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.185756…
| Accuracy difference | █▇▇▇█▅▄▃▃▃▂▂▃▂▂▂▁▂▂▃▂▂▂▂▄▂▁▂▃▁▂▂▂▃▃▄▂▂▄▃ |
| Test Accuracy | ▁▁▁▂▂▂▃▄▄▅▅▅▅▅▅▅▆▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇██████ |
| Training Accuracy | ▁▁▁▂▂▂▃▄▄▄▅▅▅▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇██████ |
| Training Loss | █████▇▆▆▅▅▅▅▄▄▄▄▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁ |
| Accuracy difference | -0.02834 |
| Test Accuracy | 0.5492 |
| Training Accuracy | 0.52086 |
| Training Loss | 1.9412 |
./wandb/run-20230429_202907-83b57mw5/logs
wandb: Agent Starting Run: 6f26m2r9 with config: wandb: batch_size: 16 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 100 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230429_225045-6f26m2r9
VBox(children=(Label(value='0.005 MB of 0.020 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.222901…
| Accuracy difference | ▄▁▂▁▂▁▁▃▅▅▃▂▃▃▄▄▅▆▄▆▆▆▅▇▆▆▆▆▆▇▇▇█▇██▇███ |
| Test Accuracy | ▁▃▄▄▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇█▇████████████████ |
| Training Accuracy | ▁▂▃▄▄▄▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇██████████████ |
| Training Loss | █▆▆▅▅▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.01904 |
| Test Accuracy | 0.8057 |
| Training Accuracy | 0.82474 |
| Training Loss | 1.63699 |
./wandb/run-20230429_225045-6f26m2r9/logs
wandb: Agent Starting Run: klpc0p70 with config: wandb: batch_size: 16 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 100 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230430_011009-klpc0p70
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▂▂▆█▆▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄ |
| Test Accuracy | ▆▇█▆▃▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▄▇██▆▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Loss | ▄▂▁▁▃▇█████████████████ |
| Accuracy difference | 0.0002 |
| Test Accuracy | 0.1 |
| Training Accuracy | 0.1002 |
| Training Loss | 2.36095 |
./wandb/run-20230430_011009-klpc0p70/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
sweep = api.sweep(f"weiping-zhang/mc1-sgd-early-stop-lr-bs/sweeps/7tmfsvmo")
sweep.display(height=1024)
True
# update learning rate & batch size, train on normal dataset
config["parameters"]["learning_rate"]["values"] = [0.01]
config["parameters"]["batch_size"]["values"] = [32]
config["parameters"]["epochs"]["values"] = [500]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n_conv_layers", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: c56mufxw Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-n_conv_layers/sweeps/c56mufxw
wandb: Agent Starting Run: y047mnbg with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 500 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230430_014253-y047mnbg
| Accuracy difference | ▂▁▁▁▃▂▃▃▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇█▇▇█▇▇███ |
| Test Accuracy | ▁▃▄▅▆▆▇▇▇▇▇▇▇███████████████████████████ |
| Training Accuracy | ▁▃▄▅▅▅▆▆▆▇▇▇▇▇▇▇▇▇▇▇████████████████████ |
| Training Loss | █▆▅▅▄▄▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.08768 |
| Test Accuracy | 0.841 |
| Training Accuracy | 0.92868 |
| Training Loss | 1.53325 |
./wandb/run-20230430_014253-y047mnbg/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
run = api.run("weiping-zhang/mc1-sgd-early-stop-n-filters/y047mnbg")
run.display(height=1024)
True
# use 4 convolutional layers
class Net(nn.Module):
def __init__(self,config):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, 32, kernel_size=3)
self.conv2 = nn.Conv2d(32, 64, kernel_size=3)
self.conv3 = nn.Conv2d(64,256,kernel_size=3)
self.conv4 = nn.Conv2d(256,512,kernel_size=3)
self.relu = nn.ReLU()
self.pool = nn.MaxPool2d(2)
self.fc1 = nn.Linear(512 * 2 * 2, 128)
self.fc2 = nn.Linear(128, 10)
self.dropout = nn.Dropout(p=config['dropout_rate'])
def forward(self, x):
# CONV1 - Relu - CONV2 - Relu - Maxpool - CONV3 - Relu - Maxpool - CONV3 - Relu - Maxpool - fc1 - Relu - Dropout - fc2 - softmax
x = self.relu(self.conv1(x))
x = self.pool(self.relu(self.conv2(x)))
x = self.pool(self.relu(self.conv3(x)))
x = self.pool(self.relu(self.conv4(x)))
x = torch.flatten(x, start_dim=1)
x = self.relu(self.fc1(x))
x = self.dropout(x)
x = nn.functional.softmax(self.fc2(x), dim=1)
return x
# test on small dataset to check if the model is able to learn
config["parameters"]["data_size"]["values"] = ['small']
config["parameters"]["epochs"]["values"] = [2000]
config["parameters"]["early_stop_epochs"]["values"] = [500]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n-conv-layers_test", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: ngg9eqhr Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-n-conv-layers_test/sweeps/ngg9eqhr
wandb: Agent Starting Run: tl0dsgj4 with config: wandb: batch_size: 32 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 2000 wandb: epochs: 5000 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: regularization: none wandb: regularization_strength: 0 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230430_112821-tl0dsgj4
wandb: Ctrl + C detected. Stopping sweep.
sweep = api.sweep(f"weiping-zhang/mc1-sgd-early-stop-n-conv-layers_test/sweeps/ngg9eqhr")
sweep.display(height=1024)
True
# tune learning_rate & batch_size
config["parameters"]["data_size"]["values"] = ['normal']
config["parameters"]["epochs"]["values"] = [50]
config["parameters"]["learning_rate"]["values"] = [0.001,0.01,0.1]
config["parameters"]["batch_size"]["values"] = [128,64,32,16]
config["parameters"]["early_stop_epochs"]["values"] = [20]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-lr-bs", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
%wandb weiping-zhang/mc1-sgd-early-stop-lr-bs/reports/4-conv-layer-tune-lr-bs--Vmlldzo0NTI0NjQz -h 1024
%wandb weiping-zhang/mc1-sgd-early-stop-lr-bs/reports/4-conv-layer-lr-0-01-bs--Vmlldzo0NDg4Mjg5 -h 1024
from the report below, we could see that,
# update learning rate & batch size, train on normal dataset
config["parameters"]["learning_rate"]["values"] = [0.01]
config["parameters"]["batch_size"]["values"] = [16,32]
config["parameters"]["epochs"]["values"] = [500]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n_conv_layers", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
%wandb weiping-zhang/mc1-sgd-early-stop-n_conv_layers/reports/optimal-4-conv-layer-models--Vmlldzo0NTI0NzAz -h 1024
My final choice is between with 3 or 4 conv-layer, as their performance is much better than other three.
%wandb weiping-zhang/mc1-sgd-early-stop-n_conv_layers/reports/how-many-convolutional-layers-is-optimal---Vmlldzo0NDg3NjI1 -h 1024
In the last steps, the optimal model was using 2 fully connected layers, so i will further test with
# use 3 convolutional layers + 1 fully connected layer
class Net(nn.Module):
def __init__(self,config):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, 32, kernel_size=3)
self.conv2 = nn.Conv2d(32, 64, kernel_size=3)
self.conv3 = nn.Conv2d(64,256,kernel_size=3)
self.relu = nn.ReLU()
self.pool = nn.MaxPool2d(2)
self.fc1 = nn.Linear(256 * 6 * 6, 10)
self.dropout = nn.Dropout(p=config['dropout_rate'])
def forward(self, x):
# CONV1 - Relu - CONV2 - Relu - Maxpool - CONV3 - Relu - Maxpool - fc1 - softmax
x = self.relu(self.conv1(x))
x = self.pool(self.relu(self.conv2(x)))
x = self.pool(self.relu(self.conv3(x)))
x = torch.flatten(x, start_dim=1)
x = nn.functional.softmax(self.fc1(x), dim=1)
return x
config = {
"method": "grid",
"metric": {'name': 'Test Accuracy', 'goal': 'maximize'},
"parameters": {
"data_size":{"values":["normal"]},
"early_stop_epochs":{"values":[20]},
"out_1":{"values":[64]},
"out_2":{"values":[128]},
"out_3":{"values":[512]},
"optimizer": {"values": ["sgd"]},
"learning_rate": {"values": [0.01]},
"batch_size": {"values": [32]},
"momentum": {"values": [0]},
"epochs": {"values": [80]},
"dropout_rate": {"values": [0]}, # dropout = 0 -> drop out no neurons
"regularization": {"values": ["none"]}, # no regularization
"regularization_strength": {"values": [0]}}}
#
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n_conv-layers", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: 3yzbdzoc Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-n-conv-layers/sweeps/3yzbdzoc
wandb: Agent Starting Run: 4nf3wmdk with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: regularization: none wandb: regularization_strength: 0 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230509_222209-4nf3wmdk
| Accuracy difference | ▁▁▃▅▆▃▂▃▁▃▂▅▃▃▁▂▆▄▂▅▃▃▃▅▅▅▃▄▇█▄▅▆▄▆▅▅▅▇▅ |
| Test Accuracy | ▁▃▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Accuracy | ▁▃▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇█████████ |
| Training Loss | █▆▆▅▅▅▅▄▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.0259 |
| Test Accuracy | 0.7485 |
| Training Accuracy | 0.7226 |
| Training Loss | 1.74148 |
./wandb/run-20230509_222209-4nf3wmdk/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
run = api.run("/weiping-zhang/mc1-sgd-early-stop-n_conv_layers/runs/4nf3wmdk")
print(run.display(height=1024))
/opt/anaconda3/lib/python3.9/site-packages/IPython/core/display.py:419: UserWarning: Consider using IPython.display.IFrame instead
warnings.warn("Consider using IPython.display.IFrame instead")
True
# use 3 convolutional layers with 3 fully connected layers
class Net(nn.Module):
def __init__(self,config):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, 32, kernel_size=3)
self.conv2 = nn.Conv2d(32, 64, kernel_size=3)
self.conv3 = nn.Conv2d(64,256,kernel_size=3)
self.relu = nn.ReLU()
self.pool = nn.MaxPool2d(2)
self.fc1 = nn.Linear(256 * 6 * 6, 128)
self.fc2 = nn.Linear(128, 32)
self.fc3 = nn.Linear(32, 10)
self.dropout = nn.Dropout(p=config['dropout_rate'])
def forward(self, x):
# CONV1 - Relu - CONV2 - Relu - Maxpool - CONV3 - Relu - Maxpool - fc1 - Relu - Dropout
# - fc2 - Relu - Dropout - fc3 - softmax
x = self.relu(self.conv1(x))
x = self.pool(self.relu(self.conv2(x)))
x = self.pool(self.relu(self.conv3(x)))
x = torch.flatten(x, start_dim=1)
x = self.relu(self.fc1(x))
x = self.dropout(x)
x = self.dropout(self.relu(self.fc2(x)))
x = nn.functional.softmax(self.fc3(x), dim=1)
return x
run = api.run("/weiping-zhang/mc1-sgd-early-stop-n_conv_layers/runs/xqb9ydp4")
run.display(height=1024)
True
%wandb weiping-zhang/mc1-sgd-early-stop-n_conv_layers/reports/how-many-fully-connected-layers--Vmlldzo0NDg4NjE0 -h 1024
compare the three 3-conv models with 1 fc layer, 2 fc layers, 3 fc layers (80 epochs):
but the accuracy line with 1 fc layer is much flatter than with 2 fc layers, this means when use more epochs, the model with 2 fc layers will likely reach much better accuracy.
--> use the architecture 3 convolutional layers + 2 fc layers
When using more filters in convolutional layers,
It's important to balance the number of filters with the size of the dataset and the overall model architecture.
in the last steps, I have used 32 - 64 - 256 filters in the three convolutional layers
class Net(nn.Module):
def __init__(self,config):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, config.out_1, config.kernel_size, stride = config.stride, padding = config.padding)
self.conv2 = nn.Conv2d(config.out_1, config.out_2, config.kernel_size, stride = config.stride, padding = config.padding)
self.conv3 = nn.Conv2d(config.out_2,config.out_3,config.kernel_size, stride = config.stride, padding = config.padding)
self.relu = nn.ReLU()
if config.pool_type == 'max':
self.pool = nn.MaxPool2d(2)
elif config.pool_type == 'avg':
self.pool = nn.AvgPool2d(2)
# output size of each layer (including pooling)
self.size1 = math.floor((32 + 2 * config.padding - config.kernel_size) / config.stride + 1)
self.size2 = math.floor(math.floor((self.size1 + 2 * config.padding - config.kernel_size) / config.stride + 1)/2)
self.size3 = math.floor(math.floor((self.size2 + 2 * config.padding - config.kernel_size) / config.stride + 1)/2)
self.fc1 = nn.Linear(config.out_3*(self.size3)**2, 128)
self.fc2 = nn.Linear(128, 10)
self.dropout = nn.Dropout(p=config['dropout_rate'])
def forward(self, x):
# CONV1 - Relu - CONV2 - Relu - pool - CONV3 - Relu - pool - fc1 - Relu - Dropout - fc2 - softmax
x = self.relu(self.conv1(x))
x = self.pool(self.relu(self.conv2(x)))
x = self.pool(self.relu(self.conv3(x)))
x = torch.flatten(x, start_dim=1)
x = self.relu(self.fc1(x))
x = self.dropout(x)
# apply the softmax activation function to the output of the second fully connected layer
x = nn.functional.softmax(self.fc2(x), dim=1)
return x
config = {
"method": "grid",
"metric": {'name': 'Test Accuracy', 'goal': 'maximize'},
"parameters": {
"data_size":{"values":["small"]},
"early_stop_epochs":{"values":[200]},
"out_1":{"values":[64]},
"out_2":{"values":[128]},
"out_3":{"values":[512]},
"kernel_size":{"values":[3]},
"stride":{"values":[1]},
"padding":{"values":[0]},
"pool_type":{"values":["max"]},
"optimizer": {"values": ["sgd"]},
"learning_rate": {"values": [0.01]},
"batch_size": {"values": [32]},
"momentum": {"values": [0]},
"epochs": {"values": [2000]},
"dropout_rate": {"values": [0]}, # dropout = 0 -> drop out no neurons
"regularization": {"values": ["none"]}, # no regularization
"regularization_strength": {"values": [0]}}}
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n-filters-test", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: 6lza4xpb Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-n-filters-test/sweeps/6lza4xpb
wandb: Agent Starting Run: qw87t26p with config: wandb: batch_size: 32 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 200 wandb: epochs: 2000 wandb: input_fc1: 18432 wandb: kernel_size: 3 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 0 wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230501_002713-qw87t26p
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▁▁▁▁▁▁▁▅▅██████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▁▁▁▁▁▁▅▅██████████████████████████████ |
| Training Loss | ██▆▅▅▅▄▄▃▃▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 1.4619 |
./wandb/run-20230501_002713-qw87t26p/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
# by normal dataset
config["parameters"]["data_size"]["values"] = ['normal']
config["parameters"]["epochs"]["values"] = [300]
config["parameters"]["early_stop_epochs"]["values"] = [20]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n-filters", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: ie8vacvy Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-n-filters/sweeps/ie8vacvy
wandb: Agent Starting Run: 911sgix1 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 300 wandb: input_fc1: 18432 wandb: kernel_size: 3 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 0 wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230430_122049-911sgix1
VBox(children=(Label(value='0.004 MB of 0.005 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.793742…
| Accuracy difference | ▁▂▁▃▂▂▂▃▃▃▃▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇█▇▇███ |
| Test Accuracy | ▁▂▃▄▅▆▆▆▇▇▇▇▇▇▇▇████████████████████████ |
| Training Accuracy | ▁▂▃▄▄▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇████████████████ |
| Training Loss | █▇▆▅▅▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.07754 |
| Test Accuracy | 0.8501 |
| Training Accuracy | 0.92764 |
| Training Loss | 1.53503 |
./wandb/run-20230430_122049-911sgix1/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
# use half number of filters for each convolutional layer: 16 - 32 - 128
config["parameters"]["out_1"]["values"] = [16]
config["parameters"]["out_2"]["values"] = [32]
config["parameters"]["out_3"]["values"] = [128]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-n-filters", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: 3ed6xi0q Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-n-filters/sweeps/3ed6xi0q
wandb: Agent Starting Run: 1wja1hs8 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 300 wandb: input_fc1: 4608 wandb: kernel_size: 3 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 16 wandb: out_2: 32 wandb: out_3: 128 wandb: padding: 0 wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 wandb: WARNING WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find /Users/rheinpark/Desktop/deep learning.
/home/jovyan/work/persistent/wandb/run-20230430_194540-1wja1hs8
VBox(children=(Label(value='0.004 MB of 0.009 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.421331…
| Accuracy difference | ▁▁▂▄▃▂▂▂▃▃▄▃▄▃▅▅▅▅▅▆▆▆▆▆▆▆▆▇▆▇▆█▇▇█▇▇██▇ |
| Test Accuracy | ▁▃▄▄▅▆▆▆▆▇▇▇▇▇▇▇▇▇█▇████████████████████ |
| Training Accuracy | ▁▂▄▄▅▅▅▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇█████████████████ |
| Training Loss | █▇▅▅▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.0348 |
| Test Accuracy | 0.8149 |
| Training Accuracy | 0.8497 |
| Training Loss | 1.61161 |
./wandb/run-20230430_194540-1wja1hs8/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
%wandb weiping-zhang/mc1-sgd-early-stop-n-filters/reports/How-many-filters-in-convolutional-layers---Vmlldzo0NTI1MTAx -h 1024
kernel size: dimensions of the filter applied in each convolutional layer, I will test 2, 3, 4 (use same kernel size for all three convolutional layers)
stride: step size that the filter moves for each convolution, I will test 1, 2 (use same stride for all three convolutional layers)
padding: add extra values (usually zero) around the input tensor to ensure that the spatial information at the edges of the input is also considered during the convolution operation. I will test 0, 1, 2 (use same padding for all three convolutional layers).
config = {
"method": "grid",
"metric": {'name': 'Test Accuracy', 'goal': 'maximize'},
"parameters": {
"data_size":{"values":["normal"]},
"early_stop_epochs":{"values":[20]},
"out_1":{"values":[64]},
"out_2":{"values":[128]},
"out_3":{"values":[512]},
"kernel_size":{"values":[2,3,4]},
"stride":{"values":[1,2]},
"padding":{"values":[0,1,2]},
"pool_type":{"values":["max"]},
"optimizer": {"values": ["sgd"]},
"learning_rate": {"values": [0.01]},
"batch_size": {"values": [32]},
"momentum": {"values": [0]},
"epochs": {"values": [80]},
"dropout_rate": {"values": [0]}, # dropout = 0 -> drop out no neurons
"regularization": {"values": ["none"]}, # no regularization
"regularization_strength": {"values": [0]}}}
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-kernel-stride-pad", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: p1kesjyt Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-kernel-stride-pad/sweeps/p1kesjyt
wandb: Agent Starting Run: ea148kth with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 2 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 0 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230501_193908-ea148kth
VBox(children=(Label(value='0.004 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.183307…
| Accuracy difference | █▃▃▂▂▃▁▃▃▄▅▃▅▄▄▂▄▄▅▄▆▅▄▇▅▆▅▃▅▆▇▆▆▆▆▆█▅▆▇ |
| Test Accuracy | ▁▃▄▄▄▅▅▅▆▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇█████████████ |
| Training Accuracy | ▁▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Loss | █▇▆▆▅▅▅▄▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.0066 |
| Test Accuracy | 0.7287 |
| Training Accuracy | 0.7221 |
| Training Loss | 1.74131 |
./wandb/run-20230501_193908-ea148kth/logs
wandb: Sweep Agent: Waiting for job. wandb: Job received. wandb: Agent Starting Run: w0ona7dv with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 2 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 0 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 2 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230501_212956-w0ona7dv
VBox(children=(Label(value='0.004 MB of 0.009 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.420701…
| Accuracy difference | ▄▃▁▆▃▃▃▃▂▂▃▃▃▄▄▄▄▄▅▄▄▅▅▅▆▆▆▆▆▇▆▇▆▇█▆▇▇▇▇ |
| Test Accuracy | ▁▁▃▂▃▃▃▄▄▄▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇█▇▇█████ |
| Training Accuracy | ▁▁▂▂▂▃▃▃▄▄▄▄▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇███████ |
| Training Loss | ████▇▇▆▆▆▆▅▅▅▄▄▄▄▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁ |
| Accuracy difference | 0.0312 |
| Test Accuracy | 0.4689 |
| Training Accuracy | 0.5001 |
| Training Loss | 1.9591 |
./wandb/run-20230501_212956-w0ona7dv/logs
wandb: Sweep Agent: Waiting for job. wandb: Job received. wandb: Agent Starting Run: ouzv6wub with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 2 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230501_224611-ouzv6wub
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.230868…
| Accuracy difference | ▂▁▄▄▃▄▄▄▂▁▃▃▂▂▃▃▃▄▂▂▅▆▂▂▃█▃▃▅▃▅▄▅▃▅▅▅▅▅▅ |
| Test Accuracy | ▁▃▄▄▄▄▄▅▅▆▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇█▇█████████ |
| Training Accuracy | ▁▃▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇█████████ |
| Training Loss | █▆▆▅▅▅▅▄▄▄▄▃▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.00362 |
| Test Accuracy | 0.6462 |
| Training Accuracy | 0.64258 |
| Training Loss | 1.81947 |
./wandb/run-20230501_224611-ouzv6wub/logs
wandb: Sweep Agent: Waiting for job. wandb: Job received. wandb: Agent Starting Run: dpen14jd with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 2 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 2 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_005713-dpen14jd
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.242222…
| Accuracy difference | ▆▄█▆▅▅▄▄▄▃▃▃▃▂▂▁▂▂▂▁▄▁▁▃▄▂▃▂▁▁▃▂▂▂▂▂▃▄▃▂ |
| Test Accuracy | ▁▁▁▂▂▃▃▄▄▄▄▄▅▅▅▅▆▆▆▆▆▆▇▆▆▇▇▇▇▇▇▇▇▇██████ |
| Training Accuracy | ▁▁▂▂▂▃▃▃▄▄▄▄▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇██████ |
| Training Loss | ████▇▇▆▆▆▆▅▅▅▅▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁ |
| Accuracy difference | -0.02684 |
| Test Accuracy | 0.4745 |
| Training Accuracy | 0.44766 |
| Training Loss | 2.01163 |
./wandb/run-20230502_005713-dpen14jd/logs
wandb: Agent Starting Run: y5oyt0af with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 2 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 2 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_022218-y5oyt0af
VBox(children=(Label(value='0.004 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.183274…
| Accuracy difference | ▁▂▅▃▄▂▅▄▃▂▄▂▂▄▂▄▅▃▃▃▃▃▆▂▆▅▄▄▄▅▄▅█▅▆▄▆▆▆▅ |
| Test Accuracy | ▁▃▃▄▄▅▄▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇██▇███████ |
| Training Accuracy | ▁▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Loss | █▇▆▅▅▅▅▄▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.00798 |
| Test Accuracy | 0.7335 |
| Training Accuracy | 0.72552 |
| Training Loss | 1.73804 |
./wandb/run-20230502_022218-y5oyt0af/logs
wandb: Agent Starting Run: ubhpa9ps with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 2 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 2 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 2 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_053402-ubhpa9ps
VBox(children=(Label(value='0.004 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.183299…
| Accuracy difference | █▇▅▅▅▂▆▄▃▄▄▃▂▃▃▆▆▃▃▄▅▂▁▃▃▃▃▃▂▃▁▁▄▁▂▇▃▅▂▃ |
| Test Accuracy | ▁▁▂▂▂▂▃▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇██▇████ |
| Training Accuracy | ▁▁▁▂▂▂▃▃▄▄▅▅▅▅▅▆▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇█████ |
| Training Loss | █████▇▆▆▅▅▅▅▄▄▄▄▄▃▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁ |
| Accuracy difference | -0.03082 |
| Test Accuracy | 0.5668 |
| Training Accuracy | 0.53598 |
| Training Loss | 1.92557 |
./wandb/run-20230502_053402-ubhpa9ps/logs
wandb: Agent Starting Run: vgvtu8ts with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 3 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 0 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_072706-vgvtu8ts
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▂▁▅▂▂▅▄▂▃▂▂▂▃▄▅▄▅▄▃▄▃▇▄▄▅▅▆▇▅▆▆▆▆▆▇▆▇██▇ |
| Test Accuracy | ▁▃▃▄▄▄▄▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇████████████ |
| Training Accuracy | ▁▂▃▄▄▄▄▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Loss | █▇▆▆▅▅▅▅▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.00436 |
| Test Accuracy | 0.7793 |
| Training Accuracy | 0.77494 |
| Training Loss | 1.68891 |
./wandb/run-20230502_072706-vgvtu8ts/logs
wandb: Agent Starting Run: 5vv4wr47 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 3 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 0 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 2 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_093659-5vv4wr47
/opt/conda/lib/python3.10/site-packages/torch/nn/init.py:405: UserWarning: Initializing zero-element tensors is a no-op
warnings.warn("Initializing zero-element tensors is a no-op")
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
./wandb/run-20230502_093659-5vv4wr47/logs
Run 5vv4wr47 errored: RuntimeError('Given input size: (512x1x1). Calculated output size: (512x0x0). Output size is too small')
wandb: ERROR Run 5vv4wr47 errored: RuntimeError('Given input size: (512x1x1). Calculated output size: (512x0x0). Output size is too small')
wandb: Agent Starting Run: o3usswyl with config:
wandb: batch_size: 32
wandb: data_size: normal
wandb: dropout_rate: 0
wandb: early_stop_epochs: 20
wandb: epochs: 80
wandb: kernel_size: 3
wandb: learning_rate: 0.01
wandb: momentum: 0
wandb: optimizer: sgd
wandb: out_1: 64
wandb: out_2: 128
wandb: out_3: 512
wandb: padding: 1
wandb: pool_type: max
wandb: regularization: none
wandb: regularization_strength: 0
wandb: stride: 1
Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_093720-o3usswyl
VBox(children=(Label(value='0.004 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.183265…
| Accuracy difference | ▁▁▅▄▄▃▃▃▄▂▁▂▆▃▂▂▄▆▂▅▄▄▄▇▅▇▇▅▇█▆▆▇▇▇▇▇█▇█ |
| Test Accuracy | ▁▃▃▄▄▄▅▅▅▅▆▆▆▆▆▇▇▆▇▇▇▇▇▇▇▇▇█▇▇██████████ |
| Training Accuracy | ▁▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Loss | █▆▆▆▅▅▅▄▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.00434 |
| Test Accuracy | 0.7768 |
| Training Accuracy | 0.78114 |
| Training Loss | 1.68231 |
./wandb/run-20230502_093720-o3usswyl/logs
wandb: Agent Starting Run: lgq1fa05 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 3 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 2 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_123340-lgq1fa05
VBox(children=(Label(value='0.004 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.183311…
| Accuracy difference | █▆▆▃▅▃▄▃▁▂▅▂▂▂▁▂▇▂▃▂▃▆▂▄▅▃▇▅▇█▅▇▅▅▄▆▅▅▅▅ |
| Test Accuracy | ▁▁▂▂▂▃▄▄▄▅▅▅▅▆▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇███████ |
| Training Accuracy | ▁▁▂▁▂▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇███████ |
| Training Loss | ████▇▇▆▆▅▅▅▄▄▄▄▄▃▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁ |
| Accuracy difference | -0.01806 |
| Test Accuracy | 0.6388 |
| Training Accuracy | 0.62074 |
| Training Loss | 1.84061 |
./wandb/run-20230502_123340-lgq1fa05/logs
wandb: Agent Starting Run: xmfxs0mk with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 3 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 2 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_140530-xmfxs0mk
VBox(children=(Label(value='0.004 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.183311…
| Accuracy difference | ▆▁▄▃▄▃▂▄▂▅▁▂▂▂▁▃▂▃▆▄▅▃▅▄▄▄▆▅▅▄▇▆▇▇▇▆█▇▇▇ |
| Test Accuracy | ▁▃▄▄▄▅▅▅▅▅▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇█▇█████████████ |
| Training Accuracy | ▁▃▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Loss | █▇▆▅▅▅▅▄▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.00726 |
| Test Accuracy | 0.7715 |
| Training Accuracy | 0.77876 |
| Training Loss | 1.6852 |
./wandb/run-20230502_140530-xmfxs0mk/logs
wandb: Agent Starting Run: 87i7ffr5 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 3 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 2 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 2 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_170419-87i7ffr5
VBox(children=(Label(value='0.004 MB of 0.005 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.793764…
| Accuracy difference | ▅▁█▂▃▂▂▂▂▂▁▂▃▃▂▁▄▂▂▃▂▄▂▃▅▂▃▄▄▂▃▃▃▄▃▅▂▃▄▃ |
| Test Accuracy | ▁▂▁▃▃▄▄▄▅▅▅▆▆▆▆▆▆▆▆▆▇▆▇▇▇▇▇▇▇▇▇▇████████ |
| Training Accuracy | ▁▁▂▂▃▃▄▄▄▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇████████ |
| Training Loss | ███▇▆▆▆▅▅▅▄▄▄▄▃▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁ |
| Accuracy difference | -0.02188 |
| Test Accuracy | 0.6112 |
| Training Accuracy | 0.58932 |
| Training Loss | 1.87198 |
./wandb/run-20230502_170419-87i7ffr5/logs
wandb: Agent Starting Run: qpcy0dqh with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 0 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_182710-qpcy0dqh
| Accuracy difference | ▇▁▃▃▅▃▄▃▂▅▃▅▅▃▅▅▄▅▅▄▅▅▅▆▄▅▆▅▆▇█▆▇▆▇▇▅▆▆▇ |
| Test Accuracy | ▁▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇████ |
| Training Accuracy | ▁▃▃▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇████ |
| Training Loss | █▇▆▆▅▅▅▅▄▄▄▄▄▃▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁ |
| Accuracy difference | 0.00012 |
| Test Accuracy | 0.7597 |
| Training Accuracy | 0.75982 |
| Training Loss | 1.70484 |
./wandb/run-20230502_182710-qpcy0dqh/logs
wandb: Agent Starting Run: eunwenij with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 0 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 2 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_202704-eunwenij
VBox(children=(Label(value='0.004 MB of 0.009 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.419562…
./wandb/run-20230502_202704-eunwenij/logs
Run eunwenij errored: RuntimeError("Calculated padded input size per channel: (3 x 3). Kernel size: (4 x 4). Kernel size can't be greater than actual input size")
wandb: ERROR Run eunwenij errored: RuntimeError("Calculated padded input size per channel: (3 x 3). Kernel size: (4 x 4). Kernel size can't be greater than actual input size")
wandb: Agent Starting Run: dhlng4c8 with config:
wandb: batch_size: 32
wandb: data_size: normal
wandb: dropout_rate: 0
wandb: early_stop_epochs: 20
wandb: epochs: 80
wandb: kernel_size: 4
wandb: learning_rate: 0.01
wandb: momentum: 0
wandb: optimizer: sgd
wandb: out_1: 64
wandb: out_2: 128
wandb: out_3: 512
wandb: padding: 1
wandb: pool_type: max
wandb: regularization: none
wandb: regularization_strength: 0
wandb: stride: 1
Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_202725-dhlng4c8
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.230971…
| Accuracy difference | ▂▁▂▃▂▂▂▃▂▂▃▃▃▃▅▄▃▆▃▃▃▄▄▅▅▄▅▅▆▅▅▅▆▇▅▅▆▆█▆ |
| Test Accuracy | ▁▃▃▄▄▄▅▅▅▅▆▆▆▆▆▆▇▆▇▇▇▇▇▇▇▇▇█▇███████████ |
| Training Accuracy | ▁▃▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Loss | █▆▆▆▅▅▅▄▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.01162 |
| Test Accuracy | 0.7835 |
| Training Accuracy | 0.79512 |
| Training Loss | 1.66855 |
./wandb/run-20230502_202725-dhlng4c8/logs
wandb: Agent Starting Run: 3q2yayxz with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 2 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230502_225305-3q2yayxz
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.230914…
| Accuracy difference | ▂▃█▁▂▃▂▃▃▂▂▁▃▂▄▆▂▁▂▄▂▃▃▅▃▅▅▄▅▃▄▃▆▆▄▆▅▆▄▅ |
| Test Accuracy | ▁▁▁▃▃▄▄▄▅▅▅▆▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇███▇███████ |
| Training Accuracy | ▁▁▂▂▃▄▄▄▄▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇████████ |
| Training Loss | ███▇▆▆▅▅▅▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.00054 |
| Test Accuracy | 0.6314 |
| Training Accuracy | 0.63194 |
| Training Loss | 1.82995 |
./wandb/run-20230502_225305-3q2yayxz/logs
wandb: Agent Starting Run: gokbcy2m with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 2 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230503_001454-gokbcy2m
| Accuracy difference | ▂▁▃▃▅▄▄▃▄▄▃▄▄▅▄▃▅▄▄▄▄▅▅▅▆▅▅▆▆▇▆▇▇▇█▇█▇▇▇ |
| Test Accuracy | ▁▃▃▄▄▄▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇█▇▇▇██████████ |
| Training Accuracy | ▁▃▃▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Loss | █▆▆▅▅▅▅▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.01202 |
| Test Accuracy | 0.7816 |
| Training Accuracy | 0.79362 |
| Training Loss | 1.66955 |
./wandb/run-20230503_001454-gokbcy2m/logs
wandb: Agent Starting Run: 65ndosjg with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 2 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 2 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.0166693182332286, max=1.0))…
/home/jovyan/work/persistent/wandb/run-20230503_030202-65ndosjg
VBox(children=(Label(value='0.021 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▇▇▅▅▃▂▂▃▆▃▃▁▂▄▃▄▃▃▄▅▃▅▅▇▅▅▅▄▅▆▅▅▅▅▆▆▆▇██ |
| Test Accuracy | ▁▁▂▃▃▄▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇█████████████ |
| Training Accuracy | ▁▁▂▃▃▄▄▄▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇█████████ |
| Training Loss | ███▇▆▆▅▅▅▄▄▄▄▄▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.00354 |
| Test Accuracy | 0.6312 |
| Training Accuracy | 0.63474 |
| Training Loss | 1.82746 |
./wandb/run-20230503_030202-65ndosjg/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
sweep = api.sweep("weiping-zhang/mc1-sgd-early-stop-kernel-stride-pad/sweeps/p1kesjyt")
sweep.display(height=1024)
True
optimal kernel size = 4, stride = 1, padding = 1, because
with optimal kernel size, stride, padding, i would test the difference between average pooling and maximum pooling
# select kernel_size = 4, stride = 1, padding = 1
# test avgerage pooling
config['parameters']['kernel_size']['values']=[4] #411
config['parameters']['stride']['values']=[1]
config['parameters']['padding']['values']=[1]
config['parameters']['pool_type']['values']=['avg']
config['parameters']['regularization']['values']=['none']
config['parameters']['regularization_strength']['values']=[0]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-kernel-stride-pad", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: fve538oi Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-kernel-stride-pad/sweeps/fve538oi
wandb: Agent Starting Run: ckbwpf9e with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: avg wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230503_062828-ckbwpf9e
VBox(children=(Label(value='0.004 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.183320…
| Accuracy difference | ▁▃▃▃▅▃▅▄▃▄▅▅▅▅▅▆▅▆▅▅▅▆▆▆▅█▆▆▇▇▇▆▇▇▇█▇▇█▇ |
| Test Accuracy | ▁▃▃▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇█▇███████ |
| Training Accuracy | ▁▃▃▄▄▄▅▅▅▅▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇███████ |
| Training Loss | █▆▆▅▅▅▅▅▄▄▄▄▄▄▄▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁ |
| Accuracy difference | -0.00366 |
| Test Accuracy | 0.7016 |
| Training Accuracy | 0.69794 |
| Training Loss | 1.76498 |
./wandb/run-20230503_062828-ckbwpf9e/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
%wandb weiping-zhang/mc1-sgd-early-stop-kernel-stride-pad/reports/avg-vs-max-pooling--Vmlldzo0NDg4ODY5 -h 1024
# train with 500 epochs to check how good will the accuray arrive
config['parameters']['pool_type']['values']=['max']
config["parameters"]["epochs"]["values"] = [500]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-regularization", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: 94cghr2x Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-regularization/sweeps/94cghr2x
wandb: Agent Starting Run: tyv3xsy5 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 500 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230504_203106-tyv3xsy5
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.230971…
| Accuracy difference | ▁▁▂▂▃▃▄▄▄▄▅▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇████████ |
| Test Accuracy | ▁▃▅▆▆▇▇▇▇▇▇▇████████████████████████████ |
| Training Accuracy | ▁▃▄▅▅▆▆▆▇▇▇▇▇▇▇▇▇▇██████████████████████ |
| Training Loss | █▆▅▄▄▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.11746 |
| Test Accuracy | 0.858 |
| Training Accuracy | 0.97546 |
| Training Loss | 1.48637 |
./wandb/run-20230504_203106-tyv3xsy5/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
sweep = api.sweep("weiping-zhang/mc1-sgd-early-stop-regularization/sweeps/94cghr2x")
sweep.display(height=1024)
True
l1 regularization, also called lasso regularization.
l2 regularization, also called ridge regularization
large l1&l2 regularization strength will increase the impact of regularization, weights will be smaller (or more weights will be zero), model's capacity to fit train data will decrease, model is less overfitting, more generalized. However, if strength is too large, the model may be underfitting.
dropout regularization, randomly drop neurons during training process
Overall, l1 and l2 regularization aim to reduce overfitting by adding regularization terms to loss function to penalize large weights, where dropout regularization prevents overfitting by dropping out neurons during training process. They could be used individually or in combinationto improve model performance.
config['parameters']['regularization']['values']=['l1']
config['parameters']['regularization_strength']['values']=[0.1,0.01,0.001,0.0001]
config['parameters']['pool_type']['values']=['max']
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-regularization", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: 4trv04st Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-regularization/sweeps/4trv04st
wandb: Agent Starting Run: 5slxuh6l with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l1 wandb: regularization_strength: 0.0001 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230503_103015-5slxuh6l
VBox(children=(Label(value='0.021 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▄▁▆▄▄▄▄▄▃▃▃▅▄▄▆▄▄▄▇▃▅█▃▄▅▇▅▅▆▆▄▄▄▄▅▆▅▄▆▆ |
| Test Accuracy | ▁▃▃▄▄▅▅▆▆▆▆▆▇▇▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Accuracy | ▁▃▄▄▄▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇████████ |
| Training Loss | █▅▄▃▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.01546 |
| Test Accuracy | 0.4581 |
| Training Accuracy | 0.44264 |
| Training Loss | 2.054 |
./wandb/run-20230503_103015-5slxuh6l/logs
wandb: Agent Starting Run: rkt2u088 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l1 wandb: regularization_strength: 0.001 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230503_130025-rkt2u088
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▅▅▅▅▅▇▅▄▃█▅▅▄▄▄▃▁▃▄▄▆▆▇▂▄▆▄▇▅ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▅▅▅▅▅▇▅▄▃█▅▅▄▄▄▃▁▃▄▄▆▆▇▂▄▆▄▇▅ |
| Training Loss | █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -2e-05 |
| Test Accuracy | 0.1 |
| Training Accuracy | 0.09998 |
| Training Loss | 2.32457 |
./wandb/run-20230503_130025-rkt2u088/logs
wandb: Agent Starting Run: ivnft1y4 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l1 wandb: regularization_strength: 0.01 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230503_135430-ivnft1y4
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▆▃▇▆▃▅▄▁▄▃█▆▇▇▄▂▄▄█▄▅▇▃▆▄▄▁▆▃▅▅▇▆█▇▇▃▆▃▇ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▆▃▇▆▃▅▄▁▄▃█▆▇▇▄▂▄▄█▄▅▇▃▆▄▄▁▆▃▅▅▇▆█▇▇▃▆▃▇ |
| Training Loss | █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.00268 |
| Test Accuracy | 0.1 |
| Training Accuracy | 0.10268 |
| Training Loss | 4.50213 |
./wandb/run-20230503_135430-ivnft1y4/logs
wandb: Agent Starting Run: 0jumk0tk with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l1 wandb: regularization_strength: 0.1 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230503_153045-0jumk0tk
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▅▅▇▆▃▁▇▅▇▅█▅▆▆▂▂▄▆▆▄▇▅▇▅▆ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▅▅▇▆▃▁▇▅▇▅█▅▆▆▂▂▄▆▆▄▇▅▇▅▆ |
| Training Loss | █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.00102 |
| Test Accuracy | 0.1 |
| Training Accuracy | 0.10102 |
| Training Loss | 222.14423 |
./wandb/run-20230503_153045-0jumk0tk/logs
wandb: Agent Starting Run: zw9mefty with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l2 wandb: regularization_strength: 0.0001 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230503_162138-zw9mefty
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.231078…
| Accuracy difference | ▂▁▂▂▃▂▂▁▂▃▂▃▃▄▆▃▃▄▄▄▅▅▆▅▇▆▇▆▇▆▆▇▇▆▆██▇██ |
| Test Accuracy | ▁▃▃▄▄▄▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇████████████ |
| Training Accuracy | ▁▂▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Loss | █▇▆▆▅▅▅▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.01426 |
| Test Accuracy | 0.7823 |
| Training Accuracy | 0.79656 |
| Training Loss | 1.67367 |
./wandb/run-20230503_162138-zw9mefty/logs
wandb: Agent Starting Run: 0ryyp8gx with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l2 wandb: regularization_strength: 0.001 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230503_185443-0ryyp8gx
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▄▁▂▃▃▁▂▃▃▃▃▇▄▃▂▃▂▅▆▄▄▄▅▅▄▅▄▆▆▆▆▆▆▆▆▆▇▇█▆ |
| Test Accuracy | ▁▃▄▄▄▅▅▅▅▅▅▅▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Accuracy | ▁▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Loss | █▇▆▆▅▅▅▄▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.01102 |
| Test Accuracy | 0.7604 |
| Training Accuracy | 0.74938 |
| Training Loss | 1.75064 |
./wandb/run-20230503_185443-0ryyp8gx/logs
wandb: Agent Starting Run: enxrwrs4 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l2 wandb: regularization_strength: 0.01 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230503_212318-enxrwrs4
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.230982…
| Accuracy difference | ▆▁▆█▇▇█▇▆▇█▆▆▆▆▆▆▆▆▆▆▆▇▆▇▅▆▆▆▆▆▆▇▅▆▇▆▅▆▇ |
| Test Accuracy | ▁▃▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇█▇█▇█████▇███████ |
| Training Accuracy | ▁▁▅▅▆▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇███████████████████ |
| Training Loss | █▆▄▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.01806 |
| Test Accuracy | 0.3452 |
| Training Accuracy | 0.32714 |
| Training Loss | 2.20585 |
./wandb/run-20230503_212318-enxrwrs4/logs
wandb: Sweep Agent: Waiting for job. wandb: Job received. wandb: Agent Starting Run: sgoz6wez with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l2 wandb: regularization_strength: 0.1 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230503_235149-sgoz6wez
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | █▂▁▃▂▂▃▂▂▂▂▂▂▁▂▃▁▂▃▁▃▁▃▁ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | █▂▁▃▂▂▃▂▂▂▂▂▂▁▂▃▁▂▃▁▃▁▃▁ |
| Training Loss | █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.0027 |
| Test Accuracy | 0.1 |
| Training Accuracy | 0.0973 |
| Training Loss | 2.30309 |
./wandb/run-20230503_235149-sgoz6wez/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
# l1 regularization strength of 0.1,0.01,0.001,0.0001 seems too large, so try small values 0.0005,0.00001, 0.000001
config['parameters']['regularization_strength']['values']=[0.0005,0.00001, 0.000001]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-regularization", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
with L1 strength of 0.00001 and 0.000001, models have simliar accuracy as models without regularization.
I would train with more epochs to see a long term regularization effect with strength of 0.00001 and 0.000001
%wandb weiping-zhang/mc1-sgd-early-stop-regularization/reports/L1-regularization--Vmlldzo0NDg5Njcz -h 1024
# train with 250 epochs to check how much effect will the regularization have
config["parameters"]["epochs"]["values"] = [250]
config['parameters']['regularization']['values']=['l1']
config['parameters']['regularization_strength']['values']=[0.00001,0.000001]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-regularization", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
%wandb weiping-zhang/mc1-sgd-early-stop-regularization/reports/L1-regularization-II--Vmlldzo0NDg5OTY5 -h 1024
config['parameters']['regularization']['values']=['l2']
config['parameters']['regularization_strength']['values']=[0.1,0.01,0.001,0.0001]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-regularization", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
%wandb weiping-zhang/mc1-sgd-early-stop-regularization/reports/L2-regularization--Vmlldzo0NDg5ODA0 -h 1024
L2 strength 0.001, 0.0001 with 250 epochs
model with L2 0.0001 have almost same performance and overfitting as model without any regularization. This means this strength is too weak that the regularization effect is not helpful
%wandb weiping-zhang/mc1-sgd-early-stop-regularization/reports/L2-regularization-II--Vmlldzo0NDkwMDYy -h 1024
# dropout without l1&l2 regularization
config["parameters"]["dropout_rate"]["values"] = [0,0.1,0.3,0.5,0.7]
config["parameters"]["regularization"]["values"] = ['none']
config["parameters"]["regularization_strength"]["values"] = [0]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-regularization", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: fd1okk1w Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-regularization/sweeps/fd1okk1w
wandb: Agent Starting Run: tx7fi110 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230504_065109-tx7fi110
VBox(children=(Label(value='0.004 MB of 0.005 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.793764…
| Accuracy difference | ▄▁▂▂▂▂▁▁▁▃▁▂▂▂▃▄▄▅▂▂▃▅▃▄▄▅▅▅▅▅▆▆▆▆▇▇▇█▆▇ |
| Test Accuracy | ▁▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇████████████ |
| Training Accuracy | ▁▂▃▃▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇█████████ |
| Training Loss | █▇▆▆▅▅▅▅▄▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.01784 |
| Test Accuracy | 0.7823 |
| Training Accuracy | 0.80014 |
| Training Loss | 1.6647 |
./wandb/run-20230504_065109-tx7fi110/logs
wandb: Agent Starting Run: qnvpob18 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.1 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230504_091127-qnvpob18
VBox(children=(Label(value='0.021 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▃▁▂▂▂▃▁▂▂▂▃▄▂▃▃▄▃▃▄▄▄▄▅▄▄▄▄▅▇▅▅▆█▅▆▆▆▇▇█ |
| Test Accuracy | ▁▃▃▄▄▄▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇██▇███████████ |
| Training Accuracy | ▁▂▃▃▄▄▄▄▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇█████████ |
| Training Loss | █▆▆▆▅▅▅▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.01398 |
| Test Accuracy | 0.7641 |
| Training Accuracy | 0.77808 |
| Training Loss | 1.68604 |
./wandb/run-20230504_091127-qnvpob18/logs
wandb: Agent Starting Run: s98c9ud0 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.3 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230504_114915-s98c9ud0
VBox(children=(Label(value='0.004 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.183277…
| Accuracy difference | ▁▂▃▃▄▄▃▆▄▄▅▆▃▅▃▄▄▅▄▄▅▄▅▅▆▅▄▄▇▅▆▆▆▆▇▇█▆▆▇ |
| Test Accuracy | ▁▃▃▄▄▄▄▄▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇███████████ |
| Training Accuracy | ▁▃▃▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇█████████ |
| Training Loss | █▇▆▆▅▅▅▅▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.00044 |
| Test Accuracy | 0.7482 |
| Training Accuracy | 0.74776 |
| Training Loss | 1.71617 |
./wandb/run-20230504_114915-s98c9ud0/logs
wandb: Agent Starting Run: qcc4f9gs with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.5 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230504_144732-qcc4f9gs
VBox(children=(Label(value='0.016 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.783977…
| Accuracy difference | ▁▃▄▃▃▂▄▁▂▄▂▄▂▃▄▅▂▃▅▃▃▄▄▃▄▅▂▄▅▄▆▅▅▆▅▆▇▅▅█ |
| Test Accuracy | ▁▂▃▃▄▄▄▅▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇█████████ |
| Training Accuracy | ▁▂▃▃▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇████████ |
| Training Loss | █▇▆▆▅▅▅▅▄▄▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁ |
| Accuracy difference | -0.00666 |
| Test Accuracy | 0.7233 |
| Training Accuracy | 0.71664 |
| Training Loss | 1.74692 |
./wandb/run-20230504_144732-qcc4f9gs/logs
wandb: Agent Starting Run: 3w203qyz with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.7 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230504_173404-3w203qyz
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.231042…
| Accuracy difference | ▃▄▆▄▃▄▄▁▄▅▅▃▇▄▃▄█▆▄▇▄▅▄▄▅▆▆▃▅█▅▅▄▅▄▅▅▇▆▇ |
| Test Accuracy | ▁▃▃▄▄▄▄▅▅▅▅▅▅▅▅▆▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇████████ |
| Training Accuracy | ▁▃▃▄▄▄▄▄▅▅▅▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇███████ |
| Training Loss | █▇▆▆▆▅▅▅▅▅▄▄▄▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁ |
| Accuracy difference | -0.0188 |
| Test Accuracy | 0.7013 |
| Training Accuracy | 0.6825 |
| Training Loss | 1.78134 |
./wandb/run-20230504_173404-3w203qyz/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
%wandb weiping-zhang/mc1-sgd-early-stop-regularization/reports/dropout-I--Vmlldzo0NDg5ODUx -h 1024
# dropout regularization with 150 epochs
config["parameters"]["dropout_rate"]["values"] = [0.1,0.3,0.5,0.7]
config["parameters"]["regularization"]["values"] = ['none']
config["parameters"]["regularization_strength"]["values"] = [0]
config["parameters"]["epochs"]["values"] = [150]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-regularization", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: wzmb66sy Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-regularization/sweeps/wzmb66sy
wandb: Agent Starting Run: uxw8215b with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.1 wandb: early_stop_epochs: 20 wandb: epochs: 150 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230507_054914-uxw8215b
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.231031…
| Accuracy difference | ▁▂▃▂▃▂▂▂▃▂▃▃▃▄▃▄▄▅▅▅▅▅▅▅▆▆▆▆▇▆▆▇▇▆▇▇▇▇▇█ |
| Test Accuracy | ▁▂▃▄▄▅▅▅▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇█████████████████ |
| Training Accuracy | ▁▂▃▃▄▄▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇████████████ |
| Training Loss | █▇▆▆▅▅▅▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.05882 |
| Test Accuracy | 0.8087 |
| Training Accuracy | 0.86752 |
| Training Loss | 1.5971 |
./wandb/run-20230507_054914-uxw8215b/logs
wandb: Agent Starting Run: 06477ds0 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.3 wandb: early_stop_epochs: 20 wandb: epochs: 150 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230507_110232-06477ds0
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.231031…
| Accuracy difference | ▁▃▃▂▃▂▃▂▂▃▃▃▃▃▄▃▅▄▅▅▅▅▅▆▅▅▆▇▆▇▇▆▆▇▇▇████ |
| Test Accuracy | ▁▂▂▃▄▄▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇████████████████ |
| Training Accuracy | ▁▂▂▃▄▄▄▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇████████████ |
| Training Loss | █▇▇▆▅▅▅▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.03388 |
| Test Accuracy | 0.8044 |
| Training Accuracy | 0.83828 |
| Training Loss | 1.62527 |
./wandb/run-20230507_110232-06477ds0/logs
wandb: Agent Starting Run: 4qd7nrds with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.5 wandb: early_stop_epochs: 20 wandb: epochs: 150 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230507_161627-4qd7nrds
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.231046…
| Accuracy difference | ▁▃▂▂▃▄▂▁▂▂▂▃▃▄▃▃▃▃▅▄▆▄▇▅▅▅▇▆▆▇▇▇▇▆▇▇████ |
| Test Accuracy | ▁▂▃▃▄▄▄▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇█▇█████████████ |
| Training Accuracy | ▁▂▃▃▄▄▄▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇███████████ |
| Training Loss | █▇▆▆▅▅▅▄▄▄▄▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.01874 |
| Test Accuracy | 0.8018 |
| Training Accuracy | 0.82054 |
| Training Loss | 1.64333 |
./wandb/run-20230507_161627-4qd7nrds/logs
wandb: Agent Starting Run: pe62u3c0 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.7 wandb: early_stop_epochs: 20 wandb: epochs: 150 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230507_205212-pe62u3c0
VBox(children=(Label(value='0.004 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.183269…
| Accuracy difference | ▂▃▂▃▂▃▃▁▃▂▃▃▃▃▃▃▄▃▃▅▄▅▅▄▆▅▅▆▆▇█▅▆▇▇▇███▇ |
| Test Accuracy | ▁▂▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇█████████ |
| Training Accuracy | ▁▂▂▃▃▄▄▄▄▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇██████████ |
| Training Loss | █▇▇▆▆▅▅▅▅▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | -0.00144 |
| Test Accuracy | 0.7924 |
| Training Accuracy | 0.79096 |
| Training Loss | 1.67254 |
./wandb/run-20230507_205212-pe62u3c0/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
model with dropout rate = 0.1 has lower accuracy than model without any regularization, but the accuracy gap is almost same as without regularization. This indicates, the generalization of model with droput rate 0.1 is not improved.
with dropout rate = 0.7, the model accuracy gap is largely reduced, even below 0 after 150 epochs, this indicates, with so large rate of neurons dropout, models tend to be underfitting.
%wandb weiping-zhang/mc1-sgd-early-stop-regularization/reports/dropout-II---Vmlldzo0NDg5ODg4 -h 1024
# combine l1 and dropout regularization with 250 epochs
config["parameters"]["dropout_rate"]["values"] = [0.3,0.5]
config["parameters"]["regularization"]["values"] = ['l1']
config["parameters"]["regularization_strength"]["values"] = [0.00001]
config["parameters"]["epochs"]["values"] = [250]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-regularization", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
# combine l2 and dropout
config["parameters"]["dropout_rate"]["values"] = [0.3,0.5]
config["parameters"]["regularization"]["values"] = ['l2']
config["parameters"]["regularization_strength"]["values"] = [0.001]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-regularization", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: 1ae5ri9l Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-regularization/sweeps/1ae5ri9l
wandb: Agent Starting Run: 4njuffhd with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.3 wandb: early_stop_epochs: 20 wandb: epochs: 250 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l1 wandb: regularization_strength: 1e-05 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230508_095126-4njuffhd
VBox(children=(Label(value='0.004 MB of 0.009 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.420450…
| Accuracy difference | ▁▂▃▂▃▃▃▃▄▃▄▄▅▅▅▆▅▆▆▆▆▆▆▆█▆▇▆▆▇▇▇▇██▇▇▇█▇ |
| Test Accuracy | ▁▃▄▄▅▅▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇███████████████████ |
| Training Accuracy | ▁▃▃▄▅▅▅▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇█████████████████ |
| Training Loss | █▆▅▅▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.02884 |
| Test Accuracy | 0.8085 |
| Training Accuracy | 0.83734 |
| Training Loss | 1.72705 |
./wandb/run-20230508_095126-4njuffhd/logs
wandb: Sweep Agent: Waiting for job. wandb: Job received. wandb: Agent Starting Run: 9p748dov with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.5 wandb: early_stop_epochs: 20 wandb: epochs: 250 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l1 wandb: regularization_strength: 1e-05 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230508_175743-9p748dov
VBox(children=(Label(value='0.004 MB of 0.009 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.420450…
| Accuracy difference | ▁▃▂▃▂▃▃▄▂▃▄▃▄▄▄▄▄▄▄▅▅▅▅▅▅▅▅▆▆▅▆▆▇▆▆█▆▆▆▆ |
| Test Accuracy | ▁▃▄▄▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇██████████████████ |
| Training Accuracy | ▁▃▃▄▄▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇█████████████████ |
| Training Loss | █▆▆▅▅▄▄▃▃▃▃▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.01268 |
| Test Accuracy | 0.8046 |
| Training Accuracy | 0.81728 |
| Training Loss | 1.74872 |
./wandb/run-20230508_175743-9p748dov/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
Create sweep with ID: l50fjgbw Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-regularization/sweeps/l50fjgbw
wandb: Agent Starting Run: 51zkiagw with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.3 wandb: early_stop_epochs: 20 wandb: epochs: 250 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l2 wandb: regularization_strength: 0.001 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230509_020930-51zkiagw
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.231092…
| Accuracy difference | ▁▄▂▃▂▃▃▂▂▃▃▄▄▄▄▄▄▅▅▄▆▇▆▇▆▆▇▆▆█▆▆█▆▆▇▇▆█▇ |
| Test Accuracy | ▁▃▄▄▅▅▅▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇█▇████████████████ |
| Training Accuracy | ▁▃▃▄▄▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇█████████████████ |
| Training Loss | █▆▅▅▅▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.00788 |
| Test Accuracy | 0.8165 |
| Training Accuracy | 0.82438 |
| Training Loss | 1.6846 |
./wandb/run-20230509_020930-51zkiagw/logs
wandb: Agent Starting Run: rpu564x4 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.5 wandb: early_stop_epochs: 20 wandb: epochs: 250 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l2 wandb: regularization_strength: 0.001 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230509_103842-rpu564x4
VBox(children=(Label(value='0.004 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.194620…
| Accuracy difference | ▁▂▁▂▁▂▁▁▃▁▂▃▂▃▄▃▄▄▄▅▄▃▄▄▆▅▄▅▄▅▄▅▅▅█▆▅▅▆▅ |
| Test Accuracy | ▁▃▃▄▄▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇███▇███████████████ |
| Training Accuracy | ▁▃▃▄▄▄▅▅▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇███████████████ |
| Training Loss | █▆▆▅▅▄▄▄▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.00672 |
| Test Accuracy | 0.7983 |
| Training Accuracy | 0.80502 |
| Training Loss | 1.70378 |
./wandb/run-20230509_103842-rpu564x4/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
%wandb weiping-zhang/mc1-sgd-early-stop-regularization/reports/L1-dropout-L2-dropout--Vmlldzo0NDkwMTI5 -h 1024
this indicates that regularization has helped reducing overfitting and improved the model's generalization ability.
Overall, though the test accuracy has slightly decreased, the regularization technique has shown its effectiveness in reducing overfitting and improving the model's performance on unseen data. It is a common trade-off to sacrifice a bit of train accuracy to achieve better generalization and avoid overfitting.
# l2_0.001 + dropout 0.3
config['parameters']['regularization']['values'] = ['l2']
config['parameters']['regularization_strength']['values'] = [0.001]
config['parameters']['dropout_rate']['values'] = [0.3]
config['parameters']['epochs']['values'] = [500]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-regularization", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: kismk30b Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-regularization/sweeps/kismk30b
wandb: Agent Starting Run: 2jj7rie6 with config: wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0.3 wandb: early_stop_epochs: 20 wandb: epochs: 500 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: l2 wandb: regularization_strength: 0.001 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230510_151304-2jj7rie6
VBox(children=(Label(value='0.004 MB of 0.005 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.792972…
| Accuracy difference | ▁▁▁▁▂▂▃▂▃▃▄▄▄▄▄▇▅▅▅▆▆▆▆▆▆▇▆▆▇█▆▇▆█▆▇▇▇█▇ |
| Test Accuracy | ▁▃▄▅▅▆▆▇▇▇▇▇▇▇█▇▇███████████████████████ |
| Training Accuracy | ▁▃▄▄▅▅▆▆▆▇▇▇▇▇▇▇▇▇▇█████████████████████ |
| Training Loss | █▆▅▅▄▃▃▃▃▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.02404 |
| Test Accuracy | 0.8326 |
| Training Accuracy | 0.85664 |
| Training Loss | 1.65702 |
./wandb/run-20230510_151304-2jj7rie6/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
%wandb weiping-zhang/mc1-sgd-early-stop-regularization/reports/regularization-effect-before-VS-after--Vmlldzo0NDkwMTY4 -h 1024
class Net(nn.Module):
def __init__(self,config):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, config['out_1'], config['kernel_size'], stride = config['stride'], padding = config['padding'])
self.conv2 = nn.Conv2d(config['out_1'], config['out_2'], config['kernel_size'], stride = config['stride'], padding = config['padding'])
self.conv3 = nn.Conv2d(config['out_2'], config['out_3'], config['kernel_size'], stride = config['stride'], padding = config['padding'])
self.batch_norm = config['batch_norm']
if self.batch_norm:
self.bn1 = nn.BatchNorm2d(config['out_1'])
self.bn2 = nn.BatchNorm2d(config['out_2'])
self.bn3 = nn.BatchNorm2d(config['out_3'])
self.relu = nn.ReLU()
if config['pool_type'] == 'max':
self.pool = nn.MaxPool2d(2)
elif config['pool_type'] == 'avg':
self.pool = nn.AvgPool2d(2)
# output size of each layer (including pooling)
self.size1 = math.floor((32 + 2 * config['padding'] - config['kernel_size']) / config['stride'] + 1)
self.size2 = math.floor(math.floor((self.size1 + 2 * config['padding'] - config['kernel_size']) / config['stride'] + 1)/2)
self.size3 = math.floor(math.floor((self.size2 + 2 * config['padding'] - config['kernel_size']) / config['stride'] + 1)/2)
self.fc1 = nn.Linear(config['out_3']*(self.size3)**2, 128)
self.fc2 = nn.Linear(128, 10)
self.dropout = nn.Dropout(p=config['dropout_rate'])
def forward(self, x):
if self.batch_norm:
x = self.relu(self.bn1(self.conv1(x)))
x = self.pool(self.relu(self.bn2(self.conv2(x))))
x = self.pool(self.relu(self.bn3(self.conv3(x))))
else:
x = self.relu(self.conv1(x))
x = self.pool(self.relu(self.conv2(x)))
x = self.pool(self.relu(self.conv3(x)))
x = x.view(x.size(0), -1)
x = self.relu(self.fc1(x))
x = self.dropout(x)
x = self.fc2(x)
return x
# test with small dataset
config = {
"method": "grid",
"metric": {'name': 'Test Accuracy', 'goal': 'maximize'},
"parameters": {
"data_size":{"values":["small"]},
"early_stop_epochs":{"values":[20]},
"batch_norm":{"values":[True]},
"out_1":{"values":[64]},
"out_2":{"values":[128]},
"out_3":{"values":[512]},
"kernel_size":{"values":[4]},
"stride":{"values":[1]},
"padding":{"values":[1]},
"pool_type":{"values":["max"]},
"optimizer": {"values": ["sgd"]},
"learning_rate": {"values": [0.01]},
"batch_size": {"values": [32]},
"momentum": {"values": [0]},
"epochs": {"values": [200]},
"dropout_rate": {"values": [0]}, # dropout = 0 -> drop out no neurons
"regularization": {"values": ["none"]}, # no regularization
"regularization_strength": {"values": [0]}}}
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop-test", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: aqid82t9 Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop-test/sweeps/aqid82t9
wandb: Agent Starting Run: ozqea329 with config: wandb: batch_norm: True wandb: batch_size: 32 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 200 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.01667008746492987, max=1.0)…
/home/jovyan/work/persistent/wandb/run-20230520_080538-ozqea329
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▅▆▆████████████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▅▆▆████████████████████████████████████ |
| Training Loss | ▇▆█▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 0.00452 |
./wandb/run-20230520_080538-ozqea329/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
# apply batchnorm on normal dataset
config['parameters']['data_size']['values'] = ['normal']
config['parameters']['epochs']['values'] = [80]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: uzngj6mb Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop/sweeps/uzngj6mb
wandb: Agent Starting Run: zvsvn144 with config: wandb: batch_norm: True wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 80 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving. wandb: Currently logged in as: weiping-zhang. Use `wandb login --relogin` to force relogin
/home/jovyan/work/persistent/wandb/run-20230520_001110-zvsvn144
| Accuracy difference | ▄▁▃▃▃▄▃▄▄▅▄▄▄▆▆▅▅▅▅▅▅▅▅▅▅▅▆▆▆▆▆▆▆▇▇▆█▇▆▇ |
| Test Accuracy | ▁▅▅▆▆▆▇▇▇▇▇▇▇▇▇▇▇███████████████████▇███ |
| Training Accuracy | ▁▃▄▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇████████████ |
| Training Loss | █▆▅▄▄▄▄▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.07762 |
| Test Accuracy | 0.8546 |
| Training Accuracy | 0.93222 |
| Training Loss | 0.19352 |
./wandb/run-20230520_001110-zvsvn144/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
# with 300 epochs
config['parameters']['epochs']['values'] = [300]
sweep_id = wandb.sweep(config, project="mc1-sgd-early-stop", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: v4tdfret Sweep URL: https://wandb.ai/weiping-zhang/mc1-sgd-early-stop/sweeps/v4tdfret
wandb: Agent Starting Run: lpp3wych with config: wandb: batch_norm: True wandb: batch_size: 32 wandb: data_size: normal wandb: dropout_rate: 0 wandb: early_stop_epochs: 20 wandb: epochs: 300 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: sgd wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
/home/jovyan/work/persistent/wandb/run-20230521_130523-lpp3wych
VBox(children=(Label(value='0.004 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.183075…
| Accuracy difference | ▁▂▂▃▄▄▄▅▅▅▆▅▆▇▇▆▇▇█▇█▇▇▇▇██▇█▇██████████ |
| Test Accuracy | ▁▃▅▅▆▇▇▇▇▇▇▇▇▇▇███▇█▇███████████████████ |
| Training Accuracy | ▁▂▄▅▅▅▆▆▆▇▇▇▇▇▇▇▇███████████████████████ |
| Training Loss | █▇▅▅▄▄▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.11168 |
| Test Accuracy | 0.8835 |
| Training Accuracy | 0.99518 |
| Training Loss | 0.01383 |
./wandb/run-20230521_130523-lpp3wych/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
%wandb weiping-zhang/mc1-sgd-early-stop/reports/NormBatch--Vmlldzo0NDkwNDEz -h 1024
first moment (mean or average of the gradients) helps the optimizer to move in the right direction. It keeps a moving average of the gradients to update the parameters effectively.
second moment (variance of the gradients) helps the optimizer to adjust the learning rate based on the scale of the gradients. It keeps a moving average of the squared gradients to normalize the learning rate.
They will help the optimizer to navigate through the optimization landscape efficiently and converge faster towards an optimal solution.
# test with small dataset
config['parameters']['batch_norm']['values'] = [False]
config['parameters']['optimizer']['values'] = ['adam']
config['parameters']['data_size']['values'] = ['small']
config['parameters']['early_stop_epochs']['values'] = [50]
config['parameters']['epochs']['values'] = [200]
config['parameters']['learning_rate']['values'] = [0.0001,0.001,0.01,0.1]
config['parameters']['batch_size']['values'] = [16,32,64]
sweep_id = wandb.sweep(config, project="mc1-adam-early-stop-test", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
Create sweep with ID: cyrvyzoi Sweep URL: https://wandb.ai/weiping-zhang/mc1-adam-early-stop-test/sweeps/cyrvyzoi
wandb: Agent Starting Run: ugzhg27j with config: wandb: batch_norm: False wandb: batch_size: 16 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 50 wandb: epochs: 200 wandb: kernel_size: 4 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: adam wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.016670136316679417, max=1.0…
/home/jovyan/work/persistent/wandb/run-20230520_081159-ugzhg27j
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▁▁▅▅█████████▅█▅████▅█████▅████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▁▅▅█████████▅█▅████▅█████▅████████████ |
| Training Loss | █▄▆▃▂▁▁▁▁▁▁▁▁▁▃▁▃▁▁▁▁▅▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 0.00103 |
./wandb/run-20230520_081159-ugzhg27j/logs
wandb: Sweep Agent: Waiting for job. wandb: Job received. wandb: Agent Starting Run: 1lcly50n with config: wandb: batch_norm: False wandb: batch_size: 16 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 50 wandb: epochs: 200 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: adam wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.01667073113300527, max=1.0)…
/home/jovyan/work/persistent/wandb/run-20230520_081403-1lcly50n
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▃▅▅▅█▆▆▆▆▆▆▆███████▅▅▆▆██▅██▆█▆███▆▆█▆█ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▃▅▅▅█▆▆▆▆▆▆▆███████▅▅▆▆██▅██▆█▆███▆▆█▆█ |
| Training Loss | ▁▂▁▁▁▁▁▁▁▂▂▁▁▁▁▁▁▁▁▁▄▅▂▁▁▁█▁▁▁▁▂▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 0.0 |
./wandb/run-20230520_081403-1lcly50n/logs
wandb: Agent Starting Run: fx3anw3a with config: wandb: batch_norm: False wandb: batch_size: 16 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 50 wandb: epochs: 200 wandb: kernel_size: 4 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: adam wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.01666964531953757, max=1.0)…
/home/jovyan/work/persistent/wandb/run-20230520_081552-fx3anw3a
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▅▃▃▅▆▆▅▅▅▃██▆▅▅▆▆▆█▅██▆▆▆███▆▆▆██▅████▆ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▅▃▃▅▆▆▅▅▅▃██▆▅▅▆▆▆█▅██▆▆▆███▆▆▆██▅████▆ |
| Training Loss | ▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.75 |
| Test Accuracy | 0.0 |
| Training Accuracy | 0.75 |
| Training Loss | 549.39795 |
./wandb/run-20230520_081552-fx3anw3a/logs
wandb: Agent Starting Run: fhpuayk8 with config: wandb: batch_norm: False wandb: batch_size: 32 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 50 wandb: epochs: 200 wandb: kernel_size: 4 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: adam wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.016669730100935944, max=1.0…
/home/jovyan/work/persistent/wandb/run-20230520_081730-fhpuayk8
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▅█▅████████████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▅█▅████████████████████████████████████ |
| Training Loss | █▇▁▃▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 0.0 |
./wandb/run-20230520_081730-fhpuayk8/logs
wandb: Agent Starting Run: n8tnqyqw with config: wandb: batch_norm: False wandb: batch_size: 32 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 50 wandb: epochs: 200 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: adam wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.016668457114913812, max=1.0…
/home/jovyan/work/persistent/wandb/run-20230520_081951-n8tnqyqw
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.232617…
| Accuracy difference | ▁▁▁▁▁▅██▅▅██████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▁▁▁▅██▅▅██████████████████████████████ |
| Training Loss | █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 0.00015 |
./wandb/run-20230520_081951-n8tnqyqw/logs
wandb: Agent Starting Run: uzaky3cr with config: wandb: batch_norm: False wandb: batch_size: 32 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 50 wandb: epochs: 200 wandb: kernel_size: 4 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: adam wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.016670338332187385, max=1.0…
/home/jovyan/work/persistent/wandb/run-20230520_082314-uzaky3cr
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▃▁▁▃▃▃▃▆▆▁▃▃▆██▃▃███▆▆▃█▆▆▆▆▆▆▆▆▆▆▆▆█▆▃ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▃▁▁▃▃▃▃▆▆▁▃▃▆██▃▃███▆▆▃█▆▆▆▆▆▆▆▆▆▆▆▆█▆▃ |
| Training Loss | ▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.5 |
| Test Accuracy | 0.0 |
| Training Accuracy | 0.5 |
| Training Loss | 134.60635 |
./wandb/run-20230520_082314-uzaky3cr/logs
wandb: Agent Starting Run: y4b80qz4 with config: wandb: batch_norm: False wandb: batch_size: 64 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 50 wandb: epochs: 200 wandb: kernel_size: 4 wandb: learning_rate: 0.001 wandb: momentum: 0 wandb: optimizer: adam wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.0166697686809736, max=1.0))…
/home/jovyan/work/persistent/wandb/run-20230520_082458-y4b80qz4
VBox(children=(Label(value='0.005 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.232652…
| Accuracy difference | ▁▁█▅▅███████████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁█▅▅███████████████████████████████████ |
| Training Loss | ▃▃▂▂█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 1.0 |
| Test Accuracy | 0.0 |
| Training Accuracy | 1.0 |
| Training Loss | 0.0 |
./wandb/run-20230520_082458-y4b80qz4/logs
wandb: Agent Starting Run: dz4k0izw with config: wandb: batch_norm: False wandb: batch_size: 64 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 50 wandb: epochs: 200 wandb: kernel_size: 4 wandb: learning_rate: 0.01 wandb: momentum: 0 wandb: optimizer: adam wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.016669313000359884, max=1.0…
/home/jovyan/work/persistent/wandb/run-20230520_082816-dz4k0izw
Early stopping
VBox(children=(Label(value='0.004 MB of 0.004 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=1.0, max…
| Accuracy difference | ▁▁▁▁▁▆▆▃▆▆▆▆▃▆▆▃▃▃▆▆█▃▆▆▆▆▆▆▃▆▆▆▃▆▆▆▆▆▆▆ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | ▁▁▁▁▁▆▆▃▆▆▆▆▃▆▆▃▃▃▆▆█▃▆▆▆▆▆▆▃▆▆▆▃▆▆▆▆▆▆▆ |
| Training Loss | ▂▄▂▁▁▁▁▁▁▁▁▁▃▁▁▁▁▆▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.75 |
| Test Accuracy | 0.0 |
| Training Accuracy | 0.75 |
| Training Loss | 0.47853 |
./wandb/run-20230520_082816-dz4k0izw/logs
wandb: Agent Starting Run: ar1iai40 with config: wandb: batch_norm: False wandb: batch_size: 64 wandb: data_size: small wandb: dropout_rate: 0 wandb: early_stop_epochs: 50 wandb: epochs: 200 wandb: kernel_size: 4 wandb: learning_rate: 0.1 wandb: momentum: 0 wandb: optimizer: adam wandb: out_1: 64 wandb: out_2: 128 wandb: out_3: 512 wandb: padding: 1 wandb: pool_type: max wandb: regularization: none wandb: regularization_strength: 0 wandb: stride: 1 Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.
VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.0166688163842385, max=1.0))…
/home/jovyan/work/persistent/wandb/run-20230520_083009-ar1iai40
VBox(children=(Label(value='0.008 MB of 0.021 MB uploaded (0.000 MB deduped)\r'), FloatProgress(value=0.387822…
| Accuracy difference | █▁██▁███████████████████████████████████ |
| Test Accuracy | ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Training Accuracy | █▁██▁███████████████████████████████████ |
| Training Loss | █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ |
| Accuracy difference | 0.5 |
| Test Accuracy | 0.0 |
| Training Accuracy | 0.5 |
| Training Loss | 1.04586 |
./wandb/run-20230520_083009-ar1iai40/logs
wandb: Sweep Agent: Waiting for job. wandb: Sweep Agent: Exiting.
%wandb weiping-zhang/mc1-adam-early-stop-test/reports/test-by-small-dataset--Vmlldzo0NDg0NDIz -h 1024
when using the Adam optimizer, it is important to tune the learning rate and batch size. Although Adam adapts the learning rate automatically, finding an appropriate initial learning rate and batch size can still significantly impact the training process.
config['parameters']['data_size']['values'] = ['normal']
config['parameters']['early_stop_epochs']['values'] = [20]
config['parameters']['epochs']['values'] = [50]
config['parameters']['learning_rate']['values'] = [0.00001,0.0001,0.001,0.01]
config['parameters']['batch_size']['values'] = [16,32,64,128]
sweep_id = wandb.sweep(config, project="mc1-adam-early-stop", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
%wandb weiping-zhang/mc1-adam-early-stop/reports/Adam-tune-learning-rate-batch-size--Vmlldzo0NDg0MzQx -h 1024
# 300 epochs with optimal learning rate 0.0001 batch size 64
config['parameters']['early_stop_epochs']['values'] = [20]
config['parameters']['epochs']['values'] = [200]
config['parameters']['learning_rate']['values'] = [0.0001]
config['parameters']['batch_size']['values'] = [64]
sweep_id = wandb.sweep(config, project="mc1-adam-early-stop", entity="weiping-zhang")
wandb.agent(sweep_id, function=lambda: cnn_train_evaluation(config))
%wandb weiping-zhang/mc1-adam-early-stop/reports/Adam-vs-SGD--Vmlldzo0NDkwNDgy -h 1024
The best model with sgd optimizer is a
Adam optimizer is efficient due to its adaptive learning rate to each parameter